71 lines
3.5 KiB
Python
71 lines
3.5 KiB
Python
|
import scrapy
|
||
|
|
import re
|
||
|
|
from scrapy_splash import SplashRequest
|
||
|
|
import dateparser
|
||
|
|
|
||
|
|
class ExploreSpider(scrapy.Spider):
|
||
|
|
name = "explorescraper"
|
||
|
|
terms = ['usi', 'eth', 'epfl', 'lugano', 'zurich', 'basel', 'oliveto%20lario', 'homework', 'plagiarism', 'rhb', 'sbb',
|
||
|
|
'ship', 'frighten', 'fool', 'have', 'inspire', 'heal', 'master', 'terminate', 'amend', 'scratch', 'embark',
|
||
|
|
'entail', 'execute', 'consolidate', 'cash', 'round', 'isolate', 'warrant', 'signal', 'weaken', 'pin',
|
||
|
|
'march', 'desire', 'widen', 'level', 'chat', 'board', 'contend', 'invent', 'resource', 'manufacture', 'seal',
|
||
|
|
'reconsider', 'suck', 'picture', 'crash', 'transport', 'plug', 'assign', 'enquire', 'campaign', 'trap',
|
||
|
|
'surround', 'debate', 'upgrade', 'decorate', 'confer', 'accumulate', 'profit', 'file', 'inherit', 'disrupt',
|
||
|
|
'contrast', 'chuck', 'tick', 'plead', 'dip', 'subscribe', 'educate', 'divorce', 'spin', 'row', 'obscure',
|
||
|
|
'creep', 'interest', 'overlook', 'twist', 'mature', 'blend', 'revise', 'attribute', 'explode', 'dwell',
|
||
|
|
'drown', 'alleviate', 'strip', 'grade', 'revert', 'value', 'award', 'strive', 'notify', 'remedy', 'accuse',
|
||
|
|
'instruct', 'spill', 'strain', 'comprehend', 'soften', 'postpone', 'wave', 'bounce', 'stock', 'position',
|
||
|
|
'insure', 'adhere', 'cling', 'summon','pause','empty','classify']
|
||
|
|
|
||
|
|
infinite_scroll_urls = ["https://flickr.com/explore"]
|
||
|
|
|
||
|
|
start_urls = ["https://www.flickr.com/photos/tags/train",
|
||
|
|
"https://www.flickr.com/photos/tags/tree",
|
||
|
|
"https://www.flickr.com/photos/tags/outside"]
|
||
|
|
|
||
|
|
def parse_photo(self, response):
|
||
|
|
adesc = re.compile("<h2 class=\" meta-field photo-desc \">(.*)<\\/h2>", re.DOTALL)
|
||
|
|
ad = re.compile("(\\w+\\s*\\d+\\s*,\\s*\\d+)", re.MULTILINE)
|
||
|
|
|
||
|
|
photo = {}
|
||
|
|
photo["author"] = response.css("a.owner-name::text").get().strip(' \n\t')
|
||
|
|
photo["title"] = response.css("h1::text").get().strip(' \n\t')
|
||
|
|
|
||
|
|
photo["description"] = ''.join(adesc.findall(response.text)).strip(' \n\t')
|
||
|
|
date = response.css("span.date-taken-label::text").get()
|
||
|
|
date = ad.findall(date)
|
||
|
|
date = date[0]
|
||
|
|
|
||
|
|
photo["date"] = dateparser.parse(date)
|
||
|
|
if photo["date"] is not None:
|
||
|
|
photo["date"] = photo["date"].strftime("%Y-%m-%d")
|
||
|
|
else:
|
||
|
|
photo["date"] = ''
|
||
|
|
photo["img_url"] = "https:" + response.css("img.main-photo::attr('src')").get()
|
||
|
|
photo["tags"] = response.css("ul.tags-list li a[title]::text").getall()
|
||
|
|
yield photo
|
||
|
|
|
||
|
|
def start_requests(self):
|
||
|
|
for term in self.terms:
|
||
|
|
self.infinite_scroll_urls.append("https://www.flickr.com/search/?text=" + term)
|
||
|
|
|
||
|
|
for url in self.infinite_scroll_urls:
|
||
|
|
|
||
|
|
# Load Splash lua script to scroll the page a few times
|
||
|
|
script = ""
|
||
|
|
with open("infinite_scroll.lua", "r") as f:
|
||
|
|
script = f.read()
|
||
|
|
|
||
|
|
# Build Splash request
|
||
|
|
yield SplashRequest(url=url, callback=self.parse, endpoint='execute', args={
|
||
|
|
"wait": 3,
|
||
|
|
"timeout": 90,
|
||
|
|
"lua_source": script,
|
||
|
|
"headers": {'User-Agent': "Mozilla/5.0 (X11; Linux x86_64; rv:82.0) Gecko/20100101 Firefox/82.0"}
|
||
|
|
})
|
||
|
|
|
||
|
|
def parse(self, response):
|
||
|
|
for q in response.css(".photo-list-photo-interaction"):
|
||
|
|
link = q.css("a::attr('href')").get()
|
||
|
|
yield response.follow(link, self.parse_photo)
|