diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 516a35e..5dd9cec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ image: registry.gitlab.com/paddy-hack/nikola test: script: - - pip3 install python-dateutil feedparser + - pip3 install requests python-dateutil feedparser - ./import_feed.py - nikola build except: @@ -10,7 +10,7 @@ test: pages: script: - - pip3 install python-dateutil feedparser + - pip3 install requests python-dateutil feedparser - ./import_feed.py - nikola build artifacts: diff --git a/import_feed.py b/import_feed.py index a53dda8..2affb46 100755 --- a/import_feed.py +++ b/import_feed.py @@ -4,6 +4,7 @@ import dateutil.parser import io import feedparser import re +import requests from urllib.parse import urlparse @@ -11,6 +12,8 @@ def main(): non_slug_chars = re.compile(r'[^a-zA-Z0-9-_]') # Get feed + req = requests.get('https://share.osf.io/api/v2/atom/?elasticQuery=%7B%22bool%22%3A%7B%22must%22%3A%7B%22query_string%22%3A%7B%22query%22%3A%22*%22%7D%7D%2C%22filter%22%3A%5B%7B%22term%22%3A%7B%22sources%22%3A%22LIS%20Scholarship%20Archive%22%7D%7D%2C%7B%22term%22%3A%7B%22types%22%3A%22preprint%22%7D%7D%5D%7D%7D') + feed = feedparser.parse(req.content) for d in feed['entries']: # Generate file name