tre/utils
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix FFA page 1 scraping

Browse source
This commit is contained in:
trémeur 2022-07-24 13:20:06 +01:00
parent 55b4492954
commit 3c5945b87e
1 changed files with 2 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

7
promptscrape.py
View file

@ -140,10 +140,7 @@ ffaprompttext = ffapromptsoup.find(id="comments")
ffaresoup = BeautifulSoup(str(ffaprompttext), "html.parser") ffaresoup = BeautifulSoup(str(ffaprompttext), "html.parser")
ffatheprompt = ffaresoup.find_all("h4",text=True) ffatheprompt = ffaresoup.find_all("h4",text=True)
ffacent = [] ffacent = []
for each in ffatheprompt: i = 1
if "100 words of" in (str(each.get_text())):
ffacent.append(str(each.get_text()))
i = 2
while i < 8: while i < 8:
ffapromptnew = (ffaprompt + "?page=" + str(i) + "&style=light") ffapromptnew = (ffaprompt + "?page=" + str(i) + "&style=light")
ffapromptpage = requests.get(ffapromptnew) ffapromptpage = requests.get(ffapromptnew)
@ -153,7 +150,7 @@ while i < 8:
ffatheprompt = ffaresoup.find_all("h4",text=True) ffatheprompt = ffaresoup.find_all("h4",text=True)
for each in ffatheprompt: for each in ffatheprompt:
if "100 words of" in (str(each.get_text())) or "100 Words of" in (str(each.get_text())) or "100 Words Of" in (str(each.get_text())): if "100 words of" in (str(each.get_text())) or "100 Words of" in (str(each.get_text())) or "100 Words Of" in (str(each.get_text())):
if "Re:" not in (str(each.get_text())): if "Re:" not in (str(each.get_text())) and "catch-up" not in (str(each.get_text())):
ffacent.append(str(each.get_text())) ffacent.append(str(each.get_text()))
i += 1 i += 1
if ffacent: if ffacent: