Handle exceptions properly in prompt scraper
This commit is contained in:
parent
bd85f72186
commit
31f4a80ddc
1 changed files with 376 additions and 328 deletions
|
|
@ -18,6 +18,7 @@ today = int(date.today().strftime("%d"))
|
|||
month = str(date.today().strftime("%B"))
|
||||
monthstring = ".*" + month + ".*"
|
||||
|
||||
try:
|
||||
cent = "https://100words.dreamwidth.org/tag/!prompt?style=light&tag=%21prompt"
|
||||
centpage = requests.get(cent)
|
||||
centsoup = BeautifulSoup(centpage.content, "html.parser")
|
||||
|
|
@ -32,6 +33,8 @@ centprompttext = centpromptsoup.find(class_="entry-content")
|
|||
centtheprompt = centprompttext.find("strong")
|
||||
print("100words (100 words): \033[1m" + centtheprompt.text.lower() + "\033[0m (" + centprompt + ")\n")
|
||||
thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centtheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
# for this one we need to extract the right entry from a list, which may be an <ol> but may not be. also, need to use the right month, as next month’s prompts are posted in advance
|
||||
# now defunct??
|
||||
|
|
@ -65,6 +68,7 @@ thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centthepromp
|
|||
# thefile.write("- [[" + thirtyoneprompt + "][31-days]] (any): *" + thirtyonetheprompt.lower() + "*\n")
|
||||
|
||||
|
||||
try:
|
||||
ad = "https://anythingdrabble.dreamwidth.org/tag/mod!+post?style=light&tag=mod%21+post"
|
||||
adpage = requests.get(ad)
|
||||
adsoup = BeautifulSoup(adpage.content, "html.parser")
|
||||
|
|
@ -82,7 +86,10 @@ while adstrippable[-1] == " ":
|
|||
adstrippable = adstrippable[:-1]
|
||||
print("anythingdrabble (100, 200, 300, 400, or 500 words): \033[1m" + adstrippable.lower() + "\033[0m (" + adprompt + ")\n")
|
||||
thefile.write("- [[" + adprompt + "][anythingdrabble]] (100, 200, 300, 400, or 500 words): *" + adstrippable.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
dove = "https://dove-drabbles.dreamwidth.org/?style=light"
|
||||
dovepage = requests.get(dove)
|
||||
dovesoup = BeautifulSoup(dovepage.content, "html.parser")
|
||||
|
|
@ -97,7 +104,10 @@ doveprompttext = dovepromptsoup.find(class_="entry-content")
|
|||
dovetheprompt = doveprompttext.find("i")
|
||||
print("dove-drabbles (any): \033[1m" + dovetheprompt.text.lower() + "\033[0m (" + doveprompt + ")\n")
|
||||
thefile.write("- [[" + doveprompt + "][dove-drabbles]] (any): *" + dovetheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
with requests.Session() as s:
|
||||
response = s.post(login_url , data)
|
||||
zone = "https://drabble-zone.dreamwidth.org/tag/mod-post?style=light&tag=mod-post"
|
||||
|
|
@ -128,8 +138,11 @@ with requests.Session() as s:
|
|||
emotiontheprompt = emotionprompttext.find_all("span")[-1]
|
||||
print("emotion100 (100 words or a multiple of 100): \033[1m" + emotiontheprompt.text.lower() + "\033[0m (" + emotionprompt + ")\n")
|
||||
thefile.write("- [[" + emotionprompt + "][emotion100]] (100 words or a multiple of 100): *" + emotiontheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
# for this one, have to get prompts from comments
|
||||
try:
|
||||
ffa = "https://fail-fandomanon.dreamwidth.org/?style=light"
|
||||
ffapage = requests.get(ffa)
|
||||
ffasoup = BeautifulSoup(ffapage.content, "html.parser")
|
||||
|
|
@ -168,9 +181,12 @@ if ffacent:
|
|||
ffaformat = "; ".join(ffacentnew)
|
||||
print("fail-fandomanon (any): \033[1m" + ffaformat.lower() + "\033[0m (" + ffaprompt + ")\n")
|
||||
thefile.write("- [[" + ffaprompt + "][fail-fandomanon]] (any): *" + ffaformat.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
# for this one, prompts are unavailable on tuesdays and wednesdays
|
||||
try:
|
||||
weekprogress = datetime.now().weekday()
|
||||
if not 0 < weekprogress < 3:
|
||||
fandom = "https://fandomweekly.dreamwidth.org/?style=light&tag=%23challenge"
|
||||
|
|
@ -187,7 +203,10 @@ if not 0 < weekprogress < 3:
|
|||
fandomtheprompt = fandomprompttext.find("td")
|
||||
print("fandomweekly (any, competitive): \033[1m" + fandomtheprompt.text.lower() + "\033[0m (" + fandomprompt + ")\n")
|
||||
thefile.write("- [[" + fandomprompt + "][fandomweekly]] (any, competitive): *" + fandomtheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
flash = "https://fan-flashworks.dreamwidth.org/?style=light&tag=admin"
|
||||
flashpage = requests.get(flash)
|
||||
flashsoup = BeautifulSoup(flashpage.content, "html.parser")
|
||||
|
|
@ -202,7 +221,10 @@ flashprompttext = flashpromptsoup.find(class_="entry-content")
|
|||
flashtheprompt = flashprompttext.find("center")
|
||||
print("fan-flashworks (any, can’t post elsewhere until round is closed): \033[1m" + flashtheprompt.text.lower() + "\033[0m (" + flashprompt + ")\n")
|
||||
thefile.write("- [[" + flashprompt + "][fan-flashworks]] (any, can’t post elsewhere until round is closed): *" + flashtheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
femslash = "https://femslashficlets.dreamwidth.org/tag/challenges?style=light&tag=challenges"
|
||||
femslashpage = requests.get(femslash)
|
||||
femslashsoup = BeautifulSoup(femslashpage.content, "html.parser")
|
||||
|
|
@ -218,7 +240,10 @@ femslashtheprompt = femslashprompttext.find("i")
|
|||
if femslashtheprompt is not None:
|
||||
print("femslash-ficlets (100–1000 words, F/F): \033[1m" + femslashtheprompt.text.lower() + "\033[0m (" + femslashprompt + ")\n")
|
||||
thefile.write("- [[" + femslashprompt + "][femslashficlets]] (100 words or a multiple of 100): *" + femslashtheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
with requests.Session() as s:
|
||||
response = s.post(login_url , data)
|
||||
fffc = "https://fffc.dreamwidth.org/tag/!challenges?style=light&tag=%21challenges"
|
||||
|
|
@ -267,7 +292,10 @@ with requests.Session() as s:
|
|||
fffcregulartheprompt = fffcregularprompttext.find("b")
|
||||
print("fffc regular challenge (at least 100 words): \033[1m" + fffcregulartheprompt.text.lower() + "\033[0m (" + fffcregularprompt + ")\n")
|
||||
thefile.write("- [[" + fffcregularprompt + "][fffc regular challenge]] (at least 100 words): *" + fffcregulartheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
ficlet = "https://ficlet-zone.dreamwidth.org/tag/challenge+post?style=light&tag=challenge+post"
|
||||
ficletpage = requests.get(ficlet)
|
||||
ficletsoup = BeautifulSoup(ficletpage.content, "html.parser")
|
||||
|
|
@ -282,8 +310,11 @@ ficletprompttext = ficletpromptsoup.find(class_="entry-content")
|
|||
ficlettheprompt = ficletprompttext.find("a")
|
||||
print("ficlet-zone (any): \033[1m" + ficlettheprompt.text.lower() + "\033[0m (" + ficletprompt + ")\n")
|
||||
thefile.write("- [[" + ficletprompt + "][ficlet-zone]] (any): *" + ficlettheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
# first calculate the hour of the month …
|
||||
try:
|
||||
hourselapsed = (today - 1) * 24
|
||||
hourstoday = int(datetime.now().strftime("%H"))
|
||||
currenthour = (hourselapsed + hourstoday)
|
||||
|
|
@ -305,7 +336,10 @@ with requests.Session() as s:
|
|||
hourlypromptthishour = str(hourlypromptmedian[0])[5:-5]
|
||||
print("hourlyprompts (any): \033[1m" + hourlypromptthishour.lower() + "\033[0m (" + hourlyprompt + ")\n")
|
||||
thefile.write("- [[" + hourlyprompt + "][hourlyprompts]] (any): *" + hourlypromptthishour.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
if 30 > today > 21:
|
||||
ssbingo = "https://sweetandshort.dreamwidth.org/tag/challenge:+bingo?style=light&tag=challenge:+bingo"
|
||||
ssbingopage = requests.get(ssbingo)
|
||||
|
|
@ -326,7 +360,10 @@ if 30 > today > 21:
|
|||
ssbingofinal = "; ".join(ssbingoclean).lower()
|
||||
print("sweet and short bingo (up to 300 words for two prompts, up to 600 words for four prompts): \033[1m" + ssbingofinal + "\033[0m (" + ssbingoprompt + ")\n")
|
||||
thefile.write("- [[" + ssbingoprompt + "][sweet and short bingo]] (up to 300 words for two prompts, up to 600 words for four prompts): *" + ssbingofinal + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
if 16 > today > 7:
|
||||
ssquicky = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+comment+quicky?mode=and&style=light&tag=%21new+challenge,challenge:+comment+quicky"
|
||||
ssquickypage = requests.get(ssquicky)
|
||||
|
|
@ -351,7 +388,10 @@ if 16 > today > 7:
|
|||
ssquickycprompt = "; ".join(ssquickycomments)
|
||||
print("sweet and short comment quicky (up to 99 words): \033[1m" + ssquickycprompt.lower() + "\033[0m (" + ssquickyprompt + ")\n")
|
||||
thefile.write("- [[" + ssquickyprompt + "][sweet and short comment quicky]] (up to 99 words): *" + ssquickycprompt.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
ssmonthly = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+10+out+of+20?mode=and&style=light&tag=%21new+challenge,challenge:+10+out+of+20"
|
||||
ssmonthlypage = requests.get(ssmonthly)
|
||||
ssmonthlysoup = BeautifulSoup(ssmonthlypage.content, "html.parser")
|
||||
|
|
@ -370,7 +410,10 @@ ssmonthlypromptstripthree = re.sub("- ","",str(ssmonthlypromptstriptwo))
|
|||
ssmonthlypromptfinal = str(ssmonthlypromptstripthree)[2:-2]
|
||||
print("sweet and short monthly prompts (up to 300 words [0–9 prompts], up to 900 words [10–19 prompts], any [20 prompts]): \033[1m" + ssmonthlypromptfinal + "\033[0m (" + ssmonthlyprompt + ")\n")
|
||||
thefile.write("- [[" + ssmonthlyprompt + "][sweet and short monthly prompts]] (up to 300 words [0–9 prompts], up to 900 words [10–19 prompts], any [20 prompts]): *" + ssmonthlypromptfinal + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
if today > 14:
|
||||
sspicture = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+picture+prompt+fun?mode=and&style=light&tag=%21new+challenge,challenge:+picture+prompt+fun"
|
||||
sspicturepage = requests.get(sspicture)
|
||||
|
|
@ -386,7 +429,10 @@ if today > 14:
|
|||
sspictureprompttext = sspicturepromptsoup.find("h3")
|
||||
print("sweet and short picture prompts (up to 300 words): \033[1m" + sspictureprompttext.text.lower() + "\033[0m (" + sspictureprompt + ")\n")
|
||||
thefile.write("- [[" + sspictureprompt + "][sweet and short picture prompts]] (up to 300 words): *" + sspictureprompttext.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
vocab = "https://vocab-drabbles.dreamwidth.org/?style=light&tag=challenge"
|
||||
vocabpage = requests.get(vocab)
|
||||
vocabsoup = BeautifulSoup(vocabpage.content, "html.parser")
|
||||
|
|
@ -401,3 +447,5 @@ vocabprompttext = vocabpromptsoup.find(class_="entry-content")
|
|||
vocabtheprompt = vocabprompttext.find("strong")
|
||||
print("vocab-drabbles (50–500 words): \033[1m" + vocabtheprompt.text.lower() + "\033[0m (" + vocabprompt + ")\n")
|
||||
thefile.write("- [[" + vocabprompt + "][vocab-drabbles]] (50–500 words): *" + vocabtheprompt.text.lower() + "*\n")
|
||||
except:
|
||||
pass
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue