Handle exceptions properly in prompt scraper
This commit is contained in:
parent
bd85f72186
commit
31f4a80ddc
1 changed files with 376 additions and 328 deletions
340
promptscrape.py
340
promptscrape.py
|
@ -18,20 +18,23 @@ today = int(date.today().strftime("%d"))
|
||||||
month = str(date.today().strftime("%B"))
|
month = str(date.today().strftime("%B"))
|
||||||
monthstring = ".*" + month + ".*"
|
monthstring = ".*" + month + ".*"
|
||||||
|
|
||||||
cent = "https://100words.dreamwidth.org/tag/!prompt?style=light&tag=%21prompt"
|
try:
|
||||||
centpage = requests.get(cent)
|
cent = "https://100words.dreamwidth.org/tag/!prompt?style=light&tag=%21prompt"
|
||||||
centsoup = BeautifulSoup(centpage.content, "html.parser")
|
centpage = requests.get(cent)
|
||||||
centprompts = centsoup.find_all("h3", string=lambda text: "prompt:" in text.lower())
|
centsoup = BeautifulSoup(centpage.content, "html.parser")
|
||||||
centsubsoup = BeautifulSoup(str(centprompts[0]), "html.parser")
|
centprompts = centsoup.find_all("h3", string=lambda text: "prompt:" in text.lower())
|
||||||
centurl = centsubsoup.find("a")
|
centsubsoup = BeautifulSoup(str(centprompts[0]), "html.parser")
|
||||||
centprompt = (centurl["href"])
|
centurl = centsubsoup.find("a")
|
||||||
centpromptnew = (centurl["href"] + "?style=light")
|
centprompt = (centurl["href"])
|
||||||
centpromptpage = requests.get(centpromptnew)
|
centpromptnew = (centurl["href"] + "?style=light")
|
||||||
centpromptsoup = BeautifulSoup(centpromptpage.content, "html.parser")
|
centpromptpage = requests.get(centpromptnew)
|
||||||
centprompttext = centpromptsoup.find(class_="entry-content")
|
centpromptsoup = BeautifulSoup(centpromptpage.content, "html.parser")
|
||||||
centtheprompt = centprompttext.find("strong")
|
centprompttext = centpromptsoup.find(class_="entry-content")
|
||||||
print("100words (100 words): \033[1m" + centtheprompt.text.lower() + "\033[0m (" + centprompt + ")\n")
|
centtheprompt = centprompttext.find("strong")
|
||||||
thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centtheprompt.text.lower() + "*\n")
|
print("100words (100 words): \033[1m" + centtheprompt.text.lower() + "\033[0m (" + centprompt + ")\n")
|
||||||
|
thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centtheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
# for this one we need to extract the right entry from a list, which may be an <ol> but may not be. also, need to use the right month, as next month’s prompts are posted in advance
|
# for this one we need to extract the right entry from a list, which may be an <ol> but may not be. also, need to use the right month, as next month’s prompts are posted in advance
|
||||||
# now defunct??
|
# now defunct??
|
||||||
|
@ -65,40 +68,47 @@ thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centthepromp
|
||||||
# thefile.write("- [[" + thirtyoneprompt + "][31-days]] (any): *" + thirtyonetheprompt.lower() + "*\n")
|
# thefile.write("- [[" + thirtyoneprompt + "][31-days]] (any): *" + thirtyonetheprompt.lower() + "*\n")
|
||||||
|
|
||||||
|
|
||||||
ad = "https://anythingdrabble.dreamwidth.org/tag/mod!+post?style=light&tag=mod%21+post"
|
try:
|
||||||
adpage = requests.get(ad)
|
ad = "https://anythingdrabble.dreamwidth.org/tag/mod!+post?style=light&tag=mod%21+post"
|
||||||
adsoup = BeautifulSoup(adpage.content, "html.parser")
|
adpage = requests.get(ad)
|
||||||
adprompts = adsoup.find_all("h3", string=lambda text: "prompt post" in text.lower())
|
adsoup = BeautifulSoup(adpage.content, "html.parser")
|
||||||
adsubsoup = BeautifulSoup(str(adprompts[0]), "html.parser")
|
adprompts = adsoup.find_all("h3", string=lambda text: "prompt post" in text.lower())
|
||||||
adurl = adsubsoup.find("a")
|
adsubsoup = BeautifulSoup(str(adprompts[0]), "html.parser")
|
||||||
adprompt = (adurl["href"])
|
adurl = adsubsoup.find("a")
|
||||||
adpromptnew = (adurl["href"] + "?style=light")
|
adprompt = (adurl["href"])
|
||||||
adpromptpage = requests.get(adpromptnew)
|
adpromptnew = (adurl["href"] + "?style=light")
|
||||||
adpromptsoup = BeautifulSoup(adpromptpage.content, "html.parser")
|
adpromptpage = requests.get(adpromptnew)
|
||||||
adprompttext = adpromptsoup.find(class_="entry-content")
|
adpromptsoup = BeautifulSoup(adpromptpage.content, "html.parser")
|
||||||
adtheprompt = adprompttext.find("center")
|
adprompttext = adpromptsoup.find(class_="entry-content")
|
||||||
adstrippable = str(adtheprompt.text)
|
adtheprompt = adprompttext.find("center")
|
||||||
while adstrippable[-1] == " ":
|
adstrippable = str(adtheprompt.text)
|
||||||
|
while adstrippable[-1] == " ":
|
||||||
adstrippable = adstrippable[:-1]
|
adstrippable = adstrippable[:-1]
|
||||||
print("anythingdrabble (100, 200, 300, 400, or 500 words): \033[1m" + adstrippable.lower() + "\033[0m (" + adprompt + ")\n")
|
print("anythingdrabble (100, 200, 300, 400, or 500 words): \033[1m" + adstrippable.lower() + "\033[0m (" + adprompt + ")\n")
|
||||||
thefile.write("- [[" + adprompt + "][anythingdrabble]] (100, 200, 300, 400, or 500 words): *" + adstrippable.lower() + "*\n")
|
thefile.write("- [[" + adprompt + "][anythingdrabble]] (100, 200, 300, 400, or 500 words): *" + adstrippable.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
dove = "https://dove-drabbles.dreamwidth.org/?style=light"
|
try:
|
||||||
dovepage = requests.get(dove)
|
dove = "https://dove-drabbles.dreamwidth.org/?style=light"
|
||||||
dovesoup = BeautifulSoup(dovepage.content, "html.parser")
|
dovepage = requests.get(dove)
|
||||||
doveprompts = dovesoup.find_all("h3", string=lambda text: "prompt post" in text.lower())
|
dovesoup = BeautifulSoup(dovepage.content, "html.parser")
|
||||||
dovesubsoup = BeautifulSoup(str(doveprompts[0]), "html.parser")
|
doveprompts = dovesoup.find_all("h3", string=lambda text: "prompt post" in text.lower())
|
||||||
doveurl = dovesubsoup.find("a")
|
dovesubsoup = BeautifulSoup(str(doveprompts[0]), "html.parser")
|
||||||
doveprompt = (doveurl["href"])
|
doveurl = dovesubsoup.find("a")
|
||||||
dovepromptnew = (doveurl["href"] + "?style=light")
|
doveprompt = (doveurl["href"])
|
||||||
dovepromptpage = requests.get(dovepromptnew)
|
dovepromptnew = (doveurl["href"] + "?style=light")
|
||||||
dovepromptsoup = BeautifulSoup(dovepromptpage.content, "html.parser")
|
dovepromptpage = requests.get(dovepromptnew)
|
||||||
doveprompttext = dovepromptsoup.find(class_="entry-content")
|
dovepromptsoup = BeautifulSoup(dovepromptpage.content, "html.parser")
|
||||||
dovetheprompt = doveprompttext.find("i")
|
doveprompttext = dovepromptsoup.find(class_="entry-content")
|
||||||
print("dove-drabbles (any): \033[1m" + dovetheprompt.text.lower() + "\033[0m (" + doveprompt + ")\n")
|
dovetheprompt = doveprompttext.find("i")
|
||||||
thefile.write("- [[" + doveprompt + "][dove-drabbles]] (any): *" + dovetheprompt.text.lower() + "*\n")
|
print("dove-drabbles (any): \033[1m" + dovetheprompt.text.lower() + "\033[0m (" + doveprompt + ")\n")
|
||||||
|
thefile.write("- [[" + doveprompt + "][dove-drabbles]] (any): *" + dovetheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
with requests.Session() as s:
|
try:
|
||||||
|
with requests.Session() as s:
|
||||||
response = s.post(login_url , data)
|
response = s.post(login_url , data)
|
||||||
zone = "https://drabble-zone.dreamwidth.org/tag/mod-post?style=light&tag=mod-post"
|
zone = "https://drabble-zone.dreamwidth.org/tag/mod-post?style=light&tag=mod-post"
|
||||||
zonepage = s.get(zone)
|
zonepage = s.get(zone)
|
||||||
|
@ -128,25 +138,28 @@ with requests.Session() as s:
|
||||||
emotiontheprompt = emotionprompttext.find_all("span")[-1]
|
emotiontheprompt = emotionprompttext.find_all("span")[-1]
|
||||||
print("emotion100 (100 words or a multiple of 100): \033[1m" + emotiontheprompt.text.lower() + "\033[0m (" + emotionprompt + ")\n")
|
print("emotion100 (100 words or a multiple of 100): \033[1m" + emotiontheprompt.text.lower() + "\033[0m (" + emotionprompt + ")\n")
|
||||||
thefile.write("- [[" + emotionprompt + "][emotion100]] (100 words or a multiple of 100): *" + emotiontheprompt.text.lower() + "*\n")
|
thefile.write("- [[" + emotionprompt + "][emotion100]] (100 words or a multiple of 100): *" + emotiontheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
# for this one, have to get prompts from comments
|
# for this one, have to get prompts from comments
|
||||||
ffa = "https://fail-fandomanon.dreamwidth.org/?style=light"
|
try:
|
||||||
ffapage = requests.get(ffa)
|
ffa = "https://fail-fandomanon.dreamwidth.org/?style=light"
|
||||||
ffasoup = BeautifulSoup(ffapage.content, "html.parser")
|
ffapage = requests.get(ffa)
|
||||||
ffaprompts = ffasoup.find_all("h3", string=lambda text: "ffa dw post" in text.lower())
|
ffasoup = BeautifulSoup(ffapage.content, "html.parser")
|
||||||
ffapromptstrim = [x for x in ffaprompts if "Placeholder" not in str(x)]
|
ffaprompts = ffasoup.find_all("h3", string=lambda text: "ffa dw post" in text.lower())
|
||||||
ffasubsoup = BeautifulSoup(str(ffapromptstrim[0]), "html.parser")
|
ffapromptstrim = [x for x in ffaprompts if "Placeholder" not in str(x)]
|
||||||
ffaurl = ffasubsoup.find("a")
|
ffasubsoup = BeautifulSoup(str(ffapromptstrim[0]), "html.parser")
|
||||||
ffaprompt = (ffaurl["href"])
|
ffaurl = ffasubsoup.find("a")
|
||||||
ffapromptnew = (ffaprompt + "?style=light")
|
ffaprompt = (ffaurl["href"])
|
||||||
ffapromptpage = requests.get(ffapromptnew)
|
ffapromptnew = (ffaprompt + "?style=light")
|
||||||
ffapromptsoup = BeautifulSoup(ffapromptpage.content, "html.parser")
|
ffapromptpage = requests.get(ffapromptnew)
|
||||||
ffaprompttext = ffapromptsoup.find(id="comments")
|
ffapromptsoup = BeautifulSoup(ffapromptpage.content, "html.parser")
|
||||||
ffaresoup = BeautifulSoup(str(ffaprompttext), "html.parser")
|
ffaprompttext = ffapromptsoup.find(id="comments")
|
||||||
ffatheprompt = ffaresoup.find_all("h4",text=True)
|
ffaresoup = BeautifulSoup(str(ffaprompttext), "html.parser")
|
||||||
ffacent = []
|
ffatheprompt = ffaresoup.find_all("h4",text=True)
|
||||||
i = 1
|
ffacent = []
|
||||||
while i < 8:
|
i = 1
|
||||||
|
while i < 8:
|
||||||
ffapromptnew = (ffaprompt + "?page=" + str(i) + "&style=light")
|
ffapromptnew = (ffaprompt + "?page=" + str(i) + "&style=light")
|
||||||
ffapromptpage = requests.get(ffapromptnew)
|
ffapromptpage = requests.get(ffapromptnew)
|
||||||
ffapromptsoup = BeautifulSoup(ffapromptpage.content, "html.parser")
|
ffapromptsoup = BeautifulSoup(ffapromptpage.content, "html.parser")
|
||||||
|
@ -158,7 +171,7 @@ while i < 8:
|
||||||
if "Re:" not in (str(each.get_text())) and "catch-up" not in (str(each.get_text())) and "Catch-Up" not in (str(each.get_text())):
|
if "Re:" not in (str(each.get_text())) and "catch-up" not in (str(each.get_text())) and "Catch-Up" not in (str(each.get_text())):
|
||||||
ffacent.append(str(each.get_text()))
|
ffacent.append(str(each.get_text()))
|
||||||
i += 1
|
i += 1
|
||||||
if ffacent:
|
if ffacent:
|
||||||
ffacent = list(dict.fromkeys(ffacent))
|
ffacent = list(dict.fromkeys(ffacent))
|
||||||
ffacentnew = []
|
ffacentnew = []
|
||||||
for x in ffacent:
|
for x in ffacent:
|
||||||
|
@ -168,11 +181,14 @@ if ffacent:
|
||||||
ffaformat = "; ".join(ffacentnew)
|
ffaformat = "; ".join(ffacentnew)
|
||||||
print("fail-fandomanon (any): \033[1m" + ffaformat.lower() + "\033[0m (" + ffaprompt + ")\n")
|
print("fail-fandomanon (any): \033[1m" + ffaformat.lower() + "\033[0m (" + ffaprompt + ")\n")
|
||||||
thefile.write("- [[" + ffaprompt + "][fail-fandomanon]] (any): *" + ffaformat.lower() + "*\n")
|
thefile.write("- [[" + ffaprompt + "][fail-fandomanon]] (any): *" + ffaformat.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
# for this one, prompts are unavailable on tuesdays and wednesdays
|
# for this one, prompts are unavailable on tuesdays and wednesdays
|
||||||
weekprogress = datetime.now().weekday()
|
try:
|
||||||
if not 0 < weekprogress < 3:
|
weekprogress = datetime.now().weekday()
|
||||||
|
if not 0 < weekprogress < 3:
|
||||||
fandom = "https://fandomweekly.dreamwidth.org/?style=light&tag=%23challenge"
|
fandom = "https://fandomweekly.dreamwidth.org/?style=light&tag=%23challenge"
|
||||||
fandompage = requests.get(fandom)
|
fandompage = requests.get(fandom)
|
||||||
fandomsoup = BeautifulSoup(fandompage.content, "html.parser")
|
fandomsoup = BeautifulSoup(fandompage.content, "html.parser")
|
||||||
|
@ -187,39 +203,48 @@ if not 0 < weekprogress < 3:
|
||||||
fandomtheprompt = fandomprompttext.find("td")
|
fandomtheprompt = fandomprompttext.find("td")
|
||||||
print("fandomweekly (any, competitive): \033[1m" + fandomtheprompt.text.lower() + "\033[0m (" + fandomprompt + ")\n")
|
print("fandomweekly (any, competitive): \033[1m" + fandomtheprompt.text.lower() + "\033[0m (" + fandomprompt + ")\n")
|
||||||
thefile.write("- [[" + fandomprompt + "][fandomweekly]] (any, competitive): *" + fandomtheprompt.text.lower() + "*\n")
|
thefile.write("- [[" + fandomprompt + "][fandomweekly]] (any, competitive): *" + fandomtheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
flash = "https://fan-flashworks.dreamwidth.org/?style=light&tag=admin"
|
try:
|
||||||
flashpage = requests.get(flash)
|
flash = "https://fan-flashworks.dreamwidth.org/?style=light&tag=admin"
|
||||||
flashsoup = BeautifulSoup(flashpage.content, "html.parser")
|
flashpage = requests.get(flash)
|
||||||
flashprompts = flashsoup.find_all("h3", string=lambda text: "challenge" in text.lower())
|
flashsoup = BeautifulSoup(flashpage.content, "html.parser")
|
||||||
flashsubsoup = BeautifulSoup(str(flashprompts[0]), "html.parser")
|
flashprompts = flashsoup.find_all("h3", string=lambda text: "challenge" in text.lower())
|
||||||
flashurl = flashsubsoup.find("a")
|
flashsubsoup = BeautifulSoup(str(flashprompts[0]), "html.parser")
|
||||||
flashprompt = (flashurl["href"])
|
flashurl = flashsubsoup.find("a")
|
||||||
flashpromptnew = (flashurl["href"] + "?style=light")
|
flashprompt = (flashurl["href"])
|
||||||
flashpromptpage = requests.get(flashpromptnew)
|
flashpromptnew = (flashurl["href"] + "?style=light")
|
||||||
flashpromptsoup = BeautifulSoup(flashpromptpage.content, "html.parser")
|
flashpromptpage = requests.get(flashpromptnew)
|
||||||
flashprompttext = flashpromptsoup.find(class_="entry-content")
|
flashpromptsoup = BeautifulSoup(flashpromptpage.content, "html.parser")
|
||||||
flashtheprompt = flashprompttext.find("center")
|
flashprompttext = flashpromptsoup.find(class_="entry-content")
|
||||||
print("fan-flashworks (any, can’t post elsewhere until round is closed): \033[1m" + flashtheprompt.text.lower() + "\033[0m (" + flashprompt + ")\n")
|
flashtheprompt = flashprompttext.find("center")
|
||||||
thefile.write("- [[" + flashprompt + "][fan-flashworks]] (any, can’t post elsewhere until round is closed): *" + flashtheprompt.text.lower() + "*\n")
|
print("fan-flashworks (any, can’t post elsewhere until round is closed): \033[1m" + flashtheprompt.text.lower() + "\033[0m (" + flashprompt + ")\n")
|
||||||
|
thefile.write("- [[" + flashprompt + "][fan-flashworks]] (any, can’t post elsewhere until round is closed): *" + flashtheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
femslash = "https://femslashficlets.dreamwidth.org/tag/challenges?style=light&tag=challenges"
|
try:
|
||||||
femslashpage = requests.get(femslash)
|
femslash = "https://femslashficlets.dreamwidth.org/tag/challenges?style=light&tag=challenges"
|
||||||
femslashsoup = BeautifulSoup(femslashpage.content, "html.parser")
|
femslashpage = requests.get(femslash)
|
||||||
femslashprompts = femslashsoup.find_all("h3", string=lambda text: "challenge" in text.lower())
|
femslashsoup = BeautifulSoup(femslashpage.content, "html.parser")
|
||||||
femslashsubsoup = BeautifulSoup(str(femslashprompts[0]), "html.parser")
|
femslashprompts = femslashsoup.find_all("h3", string=lambda text: "challenge" in text.lower())
|
||||||
femslashurl = femslashsubsoup.find("a")
|
femslashsubsoup = BeautifulSoup(str(femslashprompts[0]), "html.parser")
|
||||||
femslashprompt = (femslashurl["href"])
|
femslashurl = femslashsubsoup.find("a")
|
||||||
femslashpromptnew = (femslashurl["href"] + "?style=light")
|
femslashprompt = (femslashurl["href"])
|
||||||
femslashpromptpage = requests.get(femslashpromptnew)
|
femslashpromptnew = (femslashurl["href"] + "?style=light")
|
||||||
femslashpromptsoup = BeautifulSoup(femslashpromptpage.content, "html.parser")
|
femslashpromptpage = requests.get(femslashpromptnew)
|
||||||
femslashprompttext = femslashpromptsoup.find(class_="entry-content")
|
femslashpromptsoup = BeautifulSoup(femslashpromptpage.content, "html.parser")
|
||||||
femslashtheprompt = femslashprompttext.find("i")
|
femslashprompttext = femslashpromptsoup.find(class_="entry-content")
|
||||||
if femslashtheprompt is not None:
|
femslashtheprompt = femslashprompttext.find("i")
|
||||||
|
if femslashtheprompt is not None:
|
||||||
print("femslash-ficlets (100–1000 words, F/F): \033[1m" + femslashtheprompt.text.lower() + "\033[0m (" + femslashprompt + ")\n")
|
print("femslash-ficlets (100–1000 words, F/F): \033[1m" + femslashtheprompt.text.lower() + "\033[0m (" + femslashprompt + ")\n")
|
||||||
thefile.write("- [[" + femslashprompt + "][femslashficlets]] (100 words or a multiple of 100): *" + femslashtheprompt.text.lower() + "*\n")
|
thefile.write("- [[" + femslashprompt + "][femslashficlets]] (100 words or a multiple of 100): *" + femslashtheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
with requests.Session() as s:
|
try:
|
||||||
|
with requests.Session() as s:
|
||||||
response = s.post(login_url , data)
|
response = s.post(login_url , data)
|
||||||
fffc = "https://fffc.dreamwidth.org/tag/!challenges?style=light&tag=%21challenges"
|
fffc = "https://fffc.dreamwidth.org/tag/!challenges?style=light&tag=%21challenges"
|
||||||
fffcpage = s.get(fffc)
|
fffcpage = s.get(fffc)
|
||||||
|
@ -267,27 +292,33 @@ with requests.Session() as s:
|
||||||
fffcregulartheprompt = fffcregularprompttext.find("b")
|
fffcregulartheprompt = fffcregularprompttext.find("b")
|
||||||
print("fffc regular challenge (at least 100 words): \033[1m" + fffcregulartheprompt.text.lower() + "\033[0m (" + fffcregularprompt + ")\n")
|
print("fffc regular challenge (at least 100 words): \033[1m" + fffcregulartheprompt.text.lower() + "\033[0m (" + fffcregularprompt + ")\n")
|
||||||
thefile.write("- [[" + fffcregularprompt + "][fffc regular challenge]] (at least 100 words): *" + fffcregulartheprompt.text.lower() + "*\n")
|
thefile.write("- [[" + fffcregularprompt + "][fffc regular challenge]] (at least 100 words): *" + fffcregulartheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
ficlet = "https://ficlet-zone.dreamwidth.org/tag/challenge+post?style=light&tag=challenge+post"
|
try:
|
||||||
ficletpage = requests.get(ficlet)
|
ficlet = "https://ficlet-zone.dreamwidth.org/tag/challenge+post?style=light&tag=challenge+post"
|
||||||
ficletsoup = BeautifulSoup(ficletpage.content, "html.parser")
|
ficletpage = requests.get(ficlet)
|
||||||
ficletprompts = ficletsoup.find_all("h3", string=lambda text: "challenge" in text.lower())
|
ficletsoup = BeautifulSoup(ficletpage.content, "html.parser")
|
||||||
ficletsubsoup = BeautifulSoup(str(ficletprompts[0]), "html.parser")
|
ficletprompts = ficletsoup.find_all("h3", string=lambda text: "challenge" in text.lower())
|
||||||
ficleturl = ficletsubsoup.find("a")
|
ficletsubsoup = BeautifulSoup(str(ficletprompts[0]), "html.parser")
|
||||||
ficletprompt = (ficleturl["href"])
|
ficleturl = ficletsubsoup.find("a")
|
||||||
ficletpromptnew = (ficleturl["href"] + "?style=light")
|
ficletprompt = (ficleturl["href"])
|
||||||
ficletpromptpage = requests.get(ficletpromptnew)
|
ficletpromptnew = (ficleturl["href"] + "?style=light")
|
||||||
ficletpromptsoup = BeautifulSoup(ficletpromptpage.content, "html.parser")
|
ficletpromptpage = requests.get(ficletpromptnew)
|
||||||
ficletprompttext = ficletpromptsoup.find(class_="entry-content")
|
ficletpromptsoup = BeautifulSoup(ficletpromptpage.content, "html.parser")
|
||||||
ficlettheprompt = ficletprompttext.find("a")
|
ficletprompttext = ficletpromptsoup.find(class_="entry-content")
|
||||||
print("ficlet-zone (any): \033[1m" + ficlettheprompt.text.lower() + "\033[0m (" + ficletprompt + ")\n")
|
ficlettheprompt = ficletprompttext.find("a")
|
||||||
thefile.write("- [[" + ficletprompt + "][ficlet-zone]] (any): *" + ficlettheprompt.text.lower() + "*\n")
|
print("ficlet-zone (any): \033[1m" + ficlettheprompt.text.lower() + "\033[0m (" + ficletprompt + ")\n")
|
||||||
|
thefile.write("- [[" + ficletprompt + "][ficlet-zone]] (any): *" + ficlettheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
# first calculate the hour of the month …
|
# first calculate the hour of the month …
|
||||||
hourselapsed = (today - 1) * 24
|
try:
|
||||||
hourstoday = int(datetime.now().strftime("%H"))
|
hourselapsed = (today - 1) * 24
|
||||||
currenthour = (hourselapsed + hourstoday)
|
hourstoday = int(datetime.now().strftime("%H"))
|
||||||
with requests.Session() as s:
|
currenthour = (hourselapsed + hourstoday)
|
||||||
|
with requests.Session() as s:
|
||||||
response = s.post(login_url , data)
|
response = s.post(login_url , data)
|
||||||
hourly = "https://hourlyprompts.dreamwidth.org/?style=light"
|
hourly = "https://hourlyprompts.dreamwidth.org/?style=light"
|
||||||
hourlypage = s.get(hourly)
|
hourlypage = s.get(hourly)
|
||||||
|
@ -305,8 +336,11 @@ with requests.Session() as s:
|
||||||
hourlypromptthishour = str(hourlypromptmedian[0])[5:-5]
|
hourlypromptthishour = str(hourlypromptmedian[0])[5:-5]
|
||||||
print("hourlyprompts (any): \033[1m" + hourlypromptthishour.lower() + "\033[0m (" + hourlyprompt + ")\n")
|
print("hourlyprompts (any): \033[1m" + hourlypromptthishour.lower() + "\033[0m (" + hourlyprompt + ")\n")
|
||||||
thefile.write("- [[" + hourlyprompt + "][hourlyprompts]] (any): *" + hourlypromptthishour.lower() + "*\n")
|
thefile.write("- [[" + hourlyprompt + "][hourlyprompts]] (any): *" + hourlypromptthishour.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
if 30 > today > 21:
|
try:
|
||||||
|
if 30 > today > 21:
|
||||||
ssbingo = "https://sweetandshort.dreamwidth.org/tag/challenge:+bingo?style=light&tag=challenge:+bingo"
|
ssbingo = "https://sweetandshort.dreamwidth.org/tag/challenge:+bingo?style=light&tag=challenge:+bingo"
|
||||||
ssbingopage = requests.get(ssbingo)
|
ssbingopage = requests.get(ssbingo)
|
||||||
ssbingosoup = BeautifulSoup(ssbingopage.content, "html.parser")
|
ssbingosoup = BeautifulSoup(ssbingopage.content, "html.parser")
|
||||||
|
@ -326,8 +360,11 @@ if 30 > today > 21:
|
||||||
ssbingofinal = "; ".join(ssbingoclean).lower()
|
ssbingofinal = "; ".join(ssbingoclean).lower()
|
||||||
print("sweet and short bingo (up to 300 words for two prompts, up to 600 words for four prompts): \033[1m" + ssbingofinal + "\033[0m (" + ssbingoprompt + ")\n")
|
print("sweet and short bingo (up to 300 words for two prompts, up to 600 words for four prompts): \033[1m" + ssbingofinal + "\033[0m (" + ssbingoprompt + ")\n")
|
||||||
thefile.write("- [[" + ssbingoprompt + "][sweet and short bingo]] (up to 300 words for two prompts, up to 600 words for four prompts): *" + ssbingofinal + "*\n")
|
thefile.write("- [[" + ssbingoprompt + "][sweet and short bingo]] (up to 300 words for two prompts, up to 600 words for four prompts): *" + ssbingofinal + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
if 16 > today > 7:
|
try:
|
||||||
|
if 16 > today > 7:
|
||||||
ssquicky = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+comment+quicky?mode=and&style=light&tag=%21new+challenge,challenge:+comment+quicky"
|
ssquicky = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+comment+quicky?mode=and&style=light&tag=%21new+challenge,challenge:+comment+quicky"
|
||||||
ssquickypage = requests.get(ssquicky)
|
ssquickypage = requests.get(ssquicky)
|
||||||
ssquickysoup = BeautifulSoup(ssquickypage.content, "html.parser")
|
ssquickysoup = BeautifulSoup(ssquickypage.content, "html.parser")
|
||||||
|
@ -351,27 +388,33 @@ if 16 > today > 7:
|
||||||
ssquickycprompt = "; ".join(ssquickycomments)
|
ssquickycprompt = "; ".join(ssquickycomments)
|
||||||
print("sweet and short comment quicky (up to 99 words): \033[1m" + ssquickycprompt.lower() + "\033[0m (" + ssquickyprompt + ")\n")
|
print("sweet and short comment quicky (up to 99 words): \033[1m" + ssquickycprompt.lower() + "\033[0m (" + ssquickyprompt + ")\n")
|
||||||
thefile.write("- [[" + ssquickyprompt + "][sweet and short comment quicky]] (up to 99 words): *" + ssquickycprompt.lower() + "*\n")
|
thefile.write("- [[" + ssquickyprompt + "][sweet and short comment quicky]] (up to 99 words): *" + ssquickycprompt.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
ssmonthly = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+10+out+of+20?mode=and&style=light&tag=%21new+challenge,challenge:+10+out+of+20"
|
try:
|
||||||
ssmonthlypage = requests.get(ssmonthly)
|
ssmonthly = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+10+out+of+20?mode=and&style=light&tag=%21new+challenge,challenge:+10+out+of+20"
|
||||||
ssmonthlysoup = BeautifulSoup(ssmonthlypage.content, "html.parser")
|
ssmonthlypage = requests.get(ssmonthly)
|
||||||
ssmonthlyprompts = ssmonthlysoup.find_all("h3")
|
ssmonthlysoup = BeautifulSoup(ssmonthlypage.content, "html.parser")
|
||||||
ssmonthlysubsoup = BeautifulSoup(str(ssmonthlyprompts[0]), "html.parser")
|
ssmonthlyprompts = ssmonthlysoup.find_all("h3")
|
||||||
ssmonthlyurl = ssmonthlysubsoup.find("a")
|
ssmonthlysubsoup = BeautifulSoup(str(ssmonthlyprompts[0]), "html.parser")
|
||||||
ssmonthlyprompt = (ssmonthlyurl["href"])
|
ssmonthlyurl = ssmonthlysubsoup.find("a")
|
||||||
ssmonthlypromptnew = (ssmonthlyurl["href"] + "?style=light")
|
ssmonthlyprompt = (ssmonthlyurl["href"])
|
||||||
ssmonthlypromptpage = requests.get(ssmonthlypromptnew)
|
ssmonthlypromptnew = (ssmonthlyurl["href"] + "?style=light")
|
||||||
ssmonthlypromptsoup = BeautifulSoup(ssmonthlypromptpage.content, "html.parser")
|
ssmonthlypromptpage = requests.get(ssmonthlypromptnew)
|
||||||
ssmonthlyprompttext = ssmonthlypromptsoup.find(class_="entry-content")
|
ssmonthlypromptsoup = BeautifulSoup(ssmonthlypromptpage.content, "html.parser")
|
||||||
ssmonthlypromptmedian = re.findall(r"<a name=\"cutid1\">.*", str(ssmonthlyprompttext))
|
ssmonthlyprompttext = ssmonthlypromptsoup.find(class_="entry-content")
|
||||||
ssmonthlypromptstripone = re.sub("<.*?>","",str(ssmonthlypromptmedian))
|
ssmonthlypromptmedian = re.findall(r"<a name=\"cutid1\">.*", str(ssmonthlyprompttext))
|
||||||
ssmonthlypromptstriptwo = re.sub("([a-z])- ","\\1; ",str(ssmonthlypromptstripone))
|
ssmonthlypromptstripone = re.sub("<.*?>","",str(ssmonthlypromptmedian))
|
||||||
ssmonthlypromptstripthree = re.sub("- ","",str(ssmonthlypromptstriptwo))
|
ssmonthlypromptstriptwo = re.sub("([a-z])- ","\\1; ",str(ssmonthlypromptstripone))
|
||||||
ssmonthlypromptfinal = str(ssmonthlypromptstripthree)[2:-2]
|
ssmonthlypromptstripthree = re.sub("- ","",str(ssmonthlypromptstriptwo))
|
||||||
print("sweet and short monthly prompts (up to 300 words [0–9 prompts], up to 900 words [10–19 prompts], any [20 prompts]): \033[1m" + ssmonthlypromptfinal + "\033[0m (" + ssmonthlyprompt + ")\n")
|
ssmonthlypromptfinal = str(ssmonthlypromptstripthree)[2:-2]
|
||||||
thefile.write("- [[" + ssmonthlyprompt + "][sweet and short monthly prompts]] (up to 300 words [0–9 prompts], up to 900 words [10–19 prompts], any [20 prompts]): *" + ssmonthlypromptfinal + "*\n")
|
print("sweet and short monthly prompts (up to 300 words [0–9 prompts], up to 900 words [10–19 prompts], any [20 prompts]): \033[1m" + ssmonthlypromptfinal + "\033[0m (" + ssmonthlyprompt + ")\n")
|
||||||
|
thefile.write("- [[" + ssmonthlyprompt + "][sweet and short monthly prompts]] (up to 300 words [0–9 prompts], up to 900 words [10–19 prompts], any [20 prompts]): *" + ssmonthlypromptfinal + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
if today > 14:
|
try:
|
||||||
|
if today > 14:
|
||||||
sspicture = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+picture+prompt+fun?mode=and&style=light&tag=%21new+challenge,challenge:+picture+prompt+fun"
|
sspicture = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+picture+prompt+fun?mode=and&style=light&tag=%21new+challenge,challenge:+picture+prompt+fun"
|
||||||
sspicturepage = requests.get(sspicture)
|
sspicturepage = requests.get(sspicture)
|
||||||
sspicturesoup = BeautifulSoup(sspicturepage.content, "html.parser")
|
sspicturesoup = BeautifulSoup(sspicturepage.content, "html.parser")
|
||||||
|
@ -386,18 +429,23 @@ if today > 14:
|
||||||
sspictureprompttext = sspicturepromptsoup.find("h3")
|
sspictureprompttext = sspicturepromptsoup.find("h3")
|
||||||
print("sweet and short picture prompts (up to 300 words): \033[1m" + sspictureprompttext.text.lower() + "\033[0m (" + sspictureprompt + ")\n")
|
print("sweet and short picture prompts (up to 300 words): \033[1m" + sspictureprompttext.text.lower() + "\033[0m (" + sspictureprompt + ")\n")
|
||||||
thefile.write("- [[" + sspictureprompt + "][sweet and short picture prompts]] (up to 300 words): *" + sspictureprompttext.text.lower() + "*\n")
|
thefile.write("- [[" + sspictureprompt + "][sweet and short picture prompts]] (up to 300 words): *" + sspictureprompttext.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
vocab = "https://vocab-drabbles.dreamwidth.org/?style=light&tag=challenge"
|
try:
|
||||||
vocabpage = requests.get(vocab)
|
vocab = "https://vocab-drabbles.dreamwidth.org/?style=light&tag=challenge"
|
||||||
vocabsoup = BeautifulSoup(vocabpage.content, "html.parser")
|
vocabpage = requests.get(vocab)
|
||||||
vocabprompts = vocabsoup.find_all("h3")
|
vocabsoup = BeautifulSoup(vocabpage.content, "html.parser")
|
||||||
vocabsubsoup = BeautifulSoup(str(vocabprompts[0]), "html.parser")
|
vocabprompts = vocabsoup.find_all("h3")
|
||||||
vocaburl = vocabsubsoup.find("a")
|
vocabsubsoup = BeautifulSoup(str(vocabprompts[0]), "html.parser")
|
||||||
vocabprompt = (vocaburl["href"])
|
vocaburl = vocabsubsoup.find("a")
|
||||||
vocabpromptnew = (vocaburl["href"] + "?style=light")
|
vocabprompt = (vocaburl["href"])
|
||||||
vocabpromptpage = requests.get(vocabpromptnew)
|
vocabpromptnew = (vocaburl["href"] + "?style=light")
|
||||||
vocabpromptsoup = BeautifulSoup(vocabpromptpage.content, "html.parser")
|
vocabpromptpage = requests.get(vocabpromptnew)
|
||||||
vocabprompttext = vocabpromptsoup.find(class_="entry-content")
|
vocabpromptsoup = BeautifulSoup(vocabpromptpage.content, "html.parser")
|
||||||
vocabtheprompt = vocabprompttext.find("strong")
|
vocabprompttext = vocabpromptsoup.find(class_="entry-content")
|
||||||
print("vocab-drabbles (50–500 words): \033[1m" + vocabtheprompt.text.lower() + "\033[0m (" + vocabprompt + ")\n")
|
vocabtheprompt = vocabprompttext.find("strong")
|
||||||
thefile.write("- [[" + vocabprompt + "][vocab-drabbles]] (50–500 words): *" + vocabtheprompt.text.lower() + "*\n")
|
print("vocab-drabbles (50–500 words): \033[1m" + vocabtheprompt.text.lower() + "\033[0m (" + vocabprompt + ")\n")
|
||||||
|
thefile.write("- [[" + vocabprompt + "][vocab-drabbles]] (50–500 words): *" + vocabtheprompt.text.lower() + "*\n")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue