Fix tests and add new

fnielsen · Nov 8, 2018 · a7a459c75cb4feff3d5bc7f7923546968e18ed73 · a7a459c
1 parent 315a27b
commit a7a459c75cb4feff3d5bc7f7923546968e18ed73
Unified Split

Showing with 28 additions and 9 deletions.

+5 −3 scholia/arxiv.py

+1 −3 scholia/network.py

+1 −1 scholia/text.py

+17 −0 tests/test_arxiv.py

+2 −2 tests/test_text.py

+2 −0 tox.ini
diff --git a/scholia/arxiv.py b/scholia/arxiv.py
@@ -29,7 +29,7 @@
 import requests
-  
-USER_AGENT = 'scholiabot 1.0'
+  
+USER_AGENT = 'Scholia'
 def get_metadata(arxiv):
@@ -75,7 +75,7 @@ def get_metadata(arxiv):
    tree = etree.HTML(response.content)
    submissions = tree.xpath('//div[@class="submission-history"]/text()')
-  
-    datetime_as_string = submissions[-2][5:30]
+  
+    datetime_as_string = submissions[-1][5:30]
    isodatetime = parse_datetime(datetime_as_string).isoformat()
    subjects = tree.xpath(
@@ -92,13 +92,15 @@ def get_metadata(arxiv):
        'arxiv': arxiv,
        'authornames': tree.xpath('//div[@class="authors"]/a/text()'),
        'full_text_url': 'https://arxiv.org/pdf/' + arxiv + '.pdf',
-  
-        'publication_date': isodatetime,
+  
+        'publication_date': isodatetime[:10],
        'title': re.sub('\s+', ' ', tree.xpath('//h1/text()')[-1].strip()),
        'arxiv_classifications': arxiv_classifications,
    }
    # Optional DOI
    doi = tree.xpath('//td[@class="tablecell doi"]/a/text()')
+  
+    if not doi:
+  
+        doi = tree.xpath('//td[@class="tablecell msc_classes"]/a/text()')
    if doi:
        metadata['doi'] = doi[0]

diff --git a/scholia/network.py b/scholia/network.py
@@ -30,9 +30,7 @@
 def write_pajek_from_sparql(filename, sparql):
-  
-    """Write Pajek network file from SPARQL query.
-  
-
-  
-    """
+  
+    """Write Pajek network file from SPARQL query."""
    column1 = 'item1'
    column2 = 'item2'

diff --git a/scholia/text.py b/scholia/text.py
@@ -210,7 +210,7 @@ def load_text_to_topic_q_text():
    """
    try:
-  
-        return pickle.load(open(TEXT_TO_TOPIC_Q_TEXT_FILENAME))
+  
+        return pickle.load(open(TEXT_TO_TOPIC_Q_TEXT_FILENAME, 'rb'))
    except IOError:
        return TextToTopicQText()

diff --git a/tests/test_arxiv.py b/tests/test_arxiv.py
@@ -0,0 +1,17 @@
+  
+"""Test scholia arxiv module."""
+  
+
+  
+
+  
+from scholia.arxiv import get_metadata
+  
+
+  
+
+  
+def test_get_metadata():
+  
+    """Text for querying arxiv and metadata extration."""
+  
+    metadata = get_metadata('1503.00759')
+  
+    assert metadata['title'] == ('A Review of Relational Machine Learning for '
+  
+                                 'Knowledge Graphs')
+  
+    assert metadata['publication_date'] == '2015-09-28'
+  
+    assert metadata['doi'] == '10.1109/JPROC.2015.2483592'
+  
+
+  
+    metadata = get_metadata('1803.04349')
+  
+    assert metadata['publication_date'] == '2018-03-05'
+  
+    assert metadata['full_text_url'] == "https://arxiv.org/pdf/1803.04349.pdf"
diff --git a/tests/test_text.py b/tests/test_text.py
@@ -1,11 +1,11 @@
 """Test scholia text module."""
-  
-from scholia.text import TextToTopicQText
+  
+from scholia.text import load_text_to_topic_q_text
 def test_text_to_topic_q_text():
    """Test for class."""
-  
-    text_to_topic_q_text = TextToTopicQText()
+  
+    text_to_topic_q_text = load_text_to_topic_q_text()
    qs = text_to_topic_q_text.text_to_topic_qs('brain')
    assert qs == ['Q1073']
diff --git a/tox.ini b/tox.ini
@@ -5,13 +5,15 @@ envlist = flake8, pydocstyle, py27, py35
 [testenv:py27]
 commands = 
    python -m pytest --doctest-modules scholia
+  
+    python -m pytest tests
 deps=
    pytest
    -rrequirements.txt
 [testenv:py35]
 commands = 
    python -m pytest --doctest-modules scholia
+  
+    python -m pytest tests
 deps=
    pytest
    -rrequirements.txt