Merge branch 'import-feed' into 'master'
Import feed into Nikola Closes #1 See merge request !1
This commit is contained in:
commit
860bf2fad9
|
@ -2,12 +2,16 @@ image: registry.gitlab.com/paddy-hack/nikola
|
|||
|
||||
test:
|
||||
script:
|
||||
- pip3 install requests python-dateutil feedparser
|
||||
- ./import_feed.py
|
||||
- nikola build
|
||||
except:
|
||||
- master
|
||||
|
||||
pages:
|
||||
script:
|
||||
- pip3 install requests python-dateutil feedparser
|
||||
- ./import_feed.py
|
||||
- nikola build
|
||||
artifacts:
|
||||
paths:
|
||||
|
|
|
@ -0,0 +1,55 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import dateutil.parser
|
||||
import io
|
||||
import feedparser
|
||||
import re
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
def main():
|
||||
non_slug_chars = re.compile(r'[^a-zA-Z0-9-_]')
|
||||
|
||||
# Get feed
|
||||
req = requests.get('https://share.osf.io/api/v2/atom/?elasticQuery=%7B%22bool%22%3A%7B%22must%22%3A%7B%22query_string%22%3A%7B%22query%22%3A%22*%22%7D%7D%2C%22filter%22%3A%5B%7B%22term%22%3A%7B%22sources%22%3A%22LIS%20Scholarship%20Archive%22%7D%7D%2C%7B%22term%22%3A%7B%22types%22%3A%22preprint%22%7D%7D%5D%7D%7D')
|
||||
feed = feedparser.parse(req.content)
|
||||
|
||||
for d in feed['entries']:
|
||||
# Generate file name
|
||||
url = urlparse(d['link']).path
|
||||
|
||||
slug = url
|
||||
while non_slug_chars.match(slug[0]):
|
||||
slug = slug[1:]
|
||||
slug = non_slug_chars.sub('_', slug)
|
||||
|
||||
file_name = 'scholarship/%s.html' % slug
|
||||
|
||||
# Parse date
|
||||
date = dateutil.parser.parse(d['date'])
|
||||
|
||||
# Open output file
|
||||
with io.open(file_name, 'w', encoding='utf-8') as fp:
|
||||
fp.write(u"""\
|
||||
<!--
|
||||
.. title: {title}
|
||||
.. slug: {slug}
|
||||
.. date: {date}
|
||||
.. link: {link}
|
||||
.. description:
|
||||
-->
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
|
||||
<p>{text}</p>
|
||||
|
||||
""".format(title=d['title'].strip(), slug=slug,
|
||||
date=date.strftime('%Y-%m-%d %H:%M:%S'),
|
||||
link=d['link'].strip(),
|
||||
text=d['summary'].strip()))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
|
@ -1,19 +0,0 @@
|
|||
<!--
|
||||
.. title: Coming Soon
|
||||
.. slug: coming-soon
|
||||
.. date: 2017-05-05 14:08:25 UTC-04:00
|
||||
.. tags:
|
||||
.. category:
|
||||
.. link: https://lissarchive.org/scholarship/coming-soon
|
||||
.. description:
|
||||
.. type: text
|
||||
-->
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
|
||||
<p>This page will be propagated by an RSS feed of new materials in <a href="https://osf.io/preprints/lissa">LISSA</a>. Check back during our soft launch to see what's being added!</p>
|
||||
|
||||
</body>
|
||||
</html>
|
Loading…
Reference in New Issue