vicky
/
lissarchive-website
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use requests instead of feedparser to download

This commit is contained in:
Remi Rampin 2017-07-18 18:18:31 -04:00
parent f80b7de2d8
commit 077c61769c
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitlab-ci.yml
View File

@ -2,7 +2,7 @@ image: registry.gitlab.com/paddy-hack/nikola
test: test:
script: script:
- pip3 install python-dateutil feedparser - pip3 install requests python-dateutil feedparser
- ./import_feed.py - ./import_feed.py
- nikola build - nikola build
except: except:
@ -10,7 +10,7 @@ test:
pages: pages:
script: script:
- pip3 install python-dateutil feedparser - pip3 install requests python-dateutil feedparser
- ./import_feed.py - ./import_feed.py
- nikola build - nikola build
artifacts: artifacts:

3
import_feed.py
View File

@ -4,6 +4,7 @@ import dateutil.parser
import io import io
import feedparser import feedparser
import re import re
import requests
from urllib.parse import urlparse from urllib.parse import urlparse
@ -11,6 +12,8 @@ def main():
non_slug_chars = re.compile(r'[^a-zA-Z0-9-_]') non_slug_chars = re.compile(r'[^a-zA-Z0-9-_]')
# Get feed # Get feed
req = requests.get('https://share.osf.io/api/v2/atom/?elasticQuery=%7B%22bool%22%3A%7B%22must%22%3A%7B%22query_string%22%3A%7B%22query%22%3A%22*%22%7D%7D%2C%22filter%22%3A%5B%7B%22term%22%3A%7B%22sources%22%3A%22LIS%20Scholarship%20Archive%22%7D%7D%2C%7B%22term%22%3A%7B%22types%22%3A%22preprint%22%7D%7D%5D%7D%7D')
feed = feedparser.parse(req.content)
for d in feed['entries']: for d in feed['entries']:
# Generate file name # Generate file name