Fix tests and add new

fnielsen · Nov 8, 2018 · a7a459c75cb4feff3d5bc7f7923546968e18ed73 · a7a459c
1 parent 315a27b
commit a7a459c75cb4feff3d5bc7f7923546968e18ed73
Unified Split

Showing with 28 additions and 9 deletions.

+5 −3 scholia/arxiv.py

+1 −3 scholia/network.py

+1 −1 scholia/text.py

+17 −0 tests/test_arxiv.py

+2 −2 tests/test_text.py

+2 −0 tox.ini
diff --git a/scholia/arxiv.py b/scholia/arxiv.py
@@ -29,7 +29,7 @@
 import requests
 
 
-USER_AGENT = 'scholiabot 1.0'
+USER_AGENT = 'Scholia'
 
 
 def get_metadata(arxiv):
@@ -75,7 +75,7 @@ def get_metadata(arxiv):
     tree = etree.HTML(response.content)
 
     submissions = tree.xpath('//div[@class="submission-history"]/text()')
-    datetime_as_string = submissions[-2][5:30]
+    datetime_as_string = submissions[-1][5:30]
     isodatetime = parse_datetime(datetime_as_string).isoformat()
 
     subjects = tree.xpath(
@@ -92,13 +92,15 @@ def get_metadata(arxiv):
         'arxiv': arxiv,
         'authornames': tree.xpath('//div[@class="authors"]/a/text()'),
         'full_text_url': 'https://arxiv.org/pdf/' + arxiv + '.pdf',
-        'publication_date': isodatetime,
+        'publication_date': isodatetime[:10],
         'title': re.sub('\s+', ' ', tree.xpath('//h1/text()')[-1].strip()),
         'arxiv_classifications': arxiv_classifications,
     }
 
     # Optional DOI
     doi = tree.xpath('//td[@class="tablecell doi"]/a/text()')
+    if not doi:
+        doi = tree.xpath('//td[@class="tablecell msc_classes"]/a/text()')
     if doi:
         metadata['doi'] = doi[0]
 

diff --git a/scholia/network.py b/scholia/network.py
@@ -30,9 +30,7 @@
 
 
 def write_pajek_from_sparql(filename, sparql):
-    """Write Pajek network file from SPARQL query.
-
-    """
+    """Write Pajek network file from SPARQL query."""
     column1 = 'item1'
     column2 = 'item2'
 

diff --git a/scholia/text.py b/scholia/text.py
@@ -210,7 +210,7 @@ def load_text_to_topic_q_text():
 
     """
     try:
-        return pickle.load(open(TEXT_TO_TOPIC_Q_TEXT_FILENAME))
+        return pickle.load(open(TEXT_TO_TOPIC_Q_TEXT_FILENAME, 'rb'))
     except IOError:
         return TextToTopicQText()
 

diff --git a/tests/test_arxiv.py b/tests/test_arxiv.py
@@ -0,0 +1,17 @@
+"""Test scholia arxiv module."""
+
+
+from scholia.arxiv import get_metadata
+
+
+def test_get_metadata():
+    """Text for querying arxiv and metadata extration."""
+    metadata = get_metadata('1503.00759')
+    assert metadata['title'] == ('A Review of Relational Machine Learning for '
+                                 'Knowledge Graphs')
+    assert metadata['publication_date'] == '2015-09-28'
+    assert metadata['doi'] == '10.1109/JPROC.2015.2483592'
+
+    metadata = get_metadata('1803.04349')
+    assert metadata['publication_date'] == '2018-03-05'
+    assert metadata['full_text_url'] == "https://arxiv.org/pdf/1803.04349.pdf"
diff --git a/tests/test_text.py b/tests/test_text.py
@@ -1,11 +1,11 @@
 """Test scholia text module."""
 
 
-from scholia.text import TextToTopicQText
+from scholia.text import load_text_to_topic_q_text
 
 
 def test_text_to_topic_q_text():
     """Test for class."""
-    text_to_topic_q_text = TextToTopicQText()
+    text_to_topic_q_text = load_text_to_topic_q_text()
     qs = text_to_topic_q_text.text_to_topic_qs('brain')
     assert qs == ['Q1073']
diff --git a/tox.ini b/tox.ini
@@ -5,13 +5,15 @@ envlist = flake8, pydocstyle, py27, py35
 [testenv:py27]
 commands = 
     python -m pytest --doctest-modules scholia
+    python -m pytest tests
 deps=
     pytest
     -rrequirements.txt
 
 [testenv:py35]
 commands = 
     python -m pytest --doctest-modules scholia
+    python -m pytest tests
 deps=
     pytest
     -rrequirements.txt