From f4621b8ce1f573411d6891ae99f9e32cb3eaacb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tr=C3=A9meur?= Date: Sun, 24 Mar 2024 12:50:30 +0000 Subject: [PATCH] Update prompt scraper to build an HTML page with the prompts --- .gitignore | 2 +- prompts.css | 47 +++++++++++++++++++++++++++++++++++++ promptscrape.py | 61 ++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 prompts.css diff --git a/.gitignore b/.gitignore index 7d6d9e7..85d00a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -ficheader.html +prompts.html prompts.org __pycache__/* \ No newline at end of file diff --git a/prompts.css b/prompts.css new file mode 100644 index 0000000..8a4ae53 --- /dev/null +++ b/prompts.css @@ -0,0 +1,47 @@ +html { + text-transform: lowercase; + color: #808080; + background-color: #d5d5ef; + /* opacity: 0.8; */ + background-image: repeating-radial-gradient( circle at 0 0, transparent 0, #d5d5ef 28px ), repeating-linear-gradient( #cfcfe755, #cfcfe7 ); + font-family: "Poppins", sans-serif; +} + +a { + color: #999999; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +body { + margin: 0 auto; + max-width: 800px; +} + +div.promptwrapper > div.promptcomm { + border: 1px solid #808080; + padding: 0 5px; +} + +div.promptcomm:first-of-type { + border-radius: 5px 5px 0 0; +} + +div.promptcomm:last-of-type { + border-radius: 0 0 5px 5px; +} + +div.timestamp { + text-align: right; +} + +.prompt { + font-weight: 700; +} + +.notes { + color: #b3b3b3; +} diff --git a/promptscrape.py b/promptscrape.py index b4000d6..bfddbe7 100644 --- a/promptscrape.py +++ b/promptscrape.py @@ -18,6 +18,8 @@ today = int(date.today().strftime("%d")) month = str(date.today().strftime("%B")) monthstring = ".*" + month + ".*" +prompts = [] + try: cent = "https://100words.dreamwidth.org/tag/!prompt?style=light&tag=%21prompt" centpage = requests.get(cent) @@ -33,6 +35,7 @@ try: centtheprompt = centprompttext.find("strong") print("100words (100 words): \033[1m" + centtheprompt.text.lower() + "\033[0m (" + centprompt + ")\n") thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centtheprompt.text.lower() + "*\n") + prompts.append({"source":"100words","type":"comm","notes":"100 words","prompt":centtheprompt.text.lower(),"url":centprompt}) except: pass @@ -88,6 +91,7 @@ try: adstrippable = adstrippable[1:] print("anythingdrabble (100, 200, 300, 400, or 500 words): \033[1m" + adstrippable.lower() + "\033[0m (" + adprompt + ")\n") thefile.write("- [[" + adprompt + "][anythingdrabble]] (100, 200, 300, 400, or 500 words): *" + adstrippable.lower() + "*\n") + prompts.append({"source":"anythingdrabble","type":"comm","notes":"100, 200, 300, 400, or 500 words","prompt":adstrippable.lower(),"url":adprompt}) except: pass @@ -106,6 +110,7 @@ try: dovetheprompt = doveprompttext.find("i") print("dove-drabbles (any): \033[1m" + dovetheprompt.text.lower() + "\033[0m (" + doveprompt + ")\n") thefile.write("- [[" + doveprompt + "][dove-drabbles]] (any): *" + dovetheprompt.text.lower() + "*\n") + prompts.append({"source":"dove-drabbles","type":"comm","notes":"any length","prompt":dovetheprompt.text.lower(),"url":doveprompt}) except: pass @@ -126,7 +131,13 @@ try: zonetheprompt = zoneprompttext.find("strong") print("drabble-zone (100 or 200 words): \033[1m" + zonetheprompt.text.lower() + "\033[0m (" + zoneprompt + ")\n") thefile.write("- [[" + zoneprompt + "][drabble-zone]] (100 or 200 words): *" + zonetheprompt.text.lower() + "*\n") - + prompts.append({"source":"drabble-zone","type":"comm","notes":"100 or 200 words","prompt":zonetheprompt.text.lower(),"url":zoneprompt}) +except: + pass + +try: + with requests.Session() as s: + response = s.post(login_url , data) emotion = "https://emotion100.dreamwidth.org/tag/*modpost?style=light&tag=%2Amodpost" emotionpage = s.get(emotion) emotionsoup = BeautifulSoup(emotionpage.content, "html.parser") @@ -141,6 +152,7 @@ try: emotiontheprompt = emotionprompttext.find_all("span")[-1] print("emotion100 (100 words or a multiple of 100): \033[1m" + emotiontheprompt.text.lower() + "\033[0m (" + emotionprompt + ")\n") thefile.write("- [[" + emotionprompt + "][emotion100]] (100 words or a multiple of 100): *" + emotiontheprompt.text.lower() + "*\n") + prompts.append({"source":"emotion100","type":"comm","notes":"100 words or a multiple of 100","prompt":emotiontheprompt.text.lower(),"url":emotionprompt}) except: pass @@ -184,6 +196,7 @@ try: ffaformat = "; ".join(ffacentnew) print("fail-fandomanon (any): \033[1m" + ffaformat.lower() + "\033[0m (" + ffaprompt + ")\n") thefile.write("- [[" + ffaprompt + "][fail-fandomanon]] (any): *" + ffaformat.lower() + "*\n") + prompts.append({"source":"fail-fandomanon","type":"comm","notes":"any length","prompt":ffacentnew,"url":ffaprompt}) except: pass @@ -206,6 +219,7 @@ try: fandomtheprompt = fandomprompttext.find("td") print("fandomweekly (any, competitive): \033[1m" + fandomtheprompt.text.lower() + "\033[0m (" + fandomprompt + ")\n") thefile.write("- [[" + fandomprompt + "][fandomweekly]] (any, competitive): *" + fandomtheprompt.text.lower() + "*\n") + prompts.append({"source":"fandomweekly","type":"comm","notes":"any length, competitive","prompt":fandomtheprompt.text.lower(),"url":fandomprompt}) except: pass @@ -224,6 +238,7 @@ try: flashtheprompt = flashprompttext.find("center") print("fan-flashworks (any, can’t post elsewhere until round is closed): \033[1m" + flashtheprompt.text.lower() + "\033[0m (" + flashprompt + ")\n") thefile.write("- [[" + flashprompt + "][fan-flashworks]] (any, can’t post elsewhere until round is closed): *" + flashtheprompt.text.lower() + "*\n") + prompts.append({"source":"fan-flashworks","type":"comm","notes":"any length, can’t post elsewhere until round is closed","prompt":flashtheprompt.text.lower(),"url":flashprompt}) except: pass @@ -264,6 +279,7 @@ try: fffclittleprompttext = fffclittlepromptsoup.find("h3") print("fffc little special (at least 100 words): \033[1m" + fffclittleprompttext.text.lower() + "\033[0m (" + fffclittleprompt + ")\n") thefile.write("- [[" + fffclittleprompt + "][fffc little special]] (at least 100 words): *" + fffclittleprompttext.text.lower() + "*\n") + prompts.append({"source":"fffc","type":"comm","notes":"at least 100 words","prompt":fffclittleprompttext.text.lower(),"url":fffclittleprompt,"challenge":"little special"}) fffcmadnessprompts = fffcsoup.find_all("h3", string=lambda text: "froday madness" in text.lower()) fffcmadnesssubsoup = BeautifulSoup(str(fffcmadnessprompts[0]), "html.parser") fffcmadnessurl = fffcmadnesssubsoup.find("a") @@ -275,6 +291,7 @@ try: fffcmadnesstheprompt = fffcmadnessprompttext.find("b") print("fffc madness (at least 2000 words): \033[1m" + fffcmadnesstheprompt.text.lower() + "\033[0m (" + fffcmadnessprompt + ")\n") thefile.write("- [[" + fffcmadnessprompt + "][fffc madness]] (at least 2000 words): *" + fffcmadnesstheprompt.text.lower() + "*\n") + prompts.append({"source":"fffc","type":"comm","notes":"at least 2000 words","prompt":fffcmadnesstheprompt.text.lower(),"url":fffcmadnessprompt,"challenge":"froday madness"}) fffcmonthlyprompts = fffcsoup.find_all("h3", string=re.compile(monthstring)) fffcmonthlysubsoup = BeautifulSoup(str(fffcmonthlyprompts[0]), "html.parser") fffcmonthlyurl = fffcmonthlysubsoup.find("a") @@ -285,6 +302,7 @@ try: fffcmonthlyprompttext = fffcmonthlypromptsoup.find("h3") print("fffc monthly special (usually at least 500 words): \033[1m" + fffcmonthlyprompttext.text.lower() + "\033[0m (" + fffcmonthlyprompt + ")\n") thefile.write("- [[" + fffcmonthlyprompt + "][fffc monthly special]] (usually at least 500 words): *" + fffcmonthlyprompttext.text.lower() + "*\n") + prompts.append({"source":"fffc","type":"comm","notes":"usually at least 500 words","prompt":fffcmonthlyprompttext.text.lower(),"url":fffcmonthlyprompt,"challenge":"monthly challenge"}) fffcregularprompts = fffcsoup.find_all("h3", string=lambda text: "regular challenge" in text.lower()) fffcregularsubsoup = BeautifulSoup(str(fffcregularprompts[0]), "html.parser") fffcregularurl = fffcregularsubsoup.find("a") @@ -296,6 +314,7 @@ try: fffcregulartheprompt = fffcregularprompttext.find("b") print("fffc regular challenge (at least 100 words): \033[1m" + fffcregulartheprompt.text.lower() + "\033[0m (" + fffcregularprompt + ")\n") thefile.write("- [[" + fffcregularprompt + "][fffc regular challenge]] (at least 100 words): *" + fffcregulartheprompt.text.lower() + "*\n") + prompts.append({"source":"fffc","type":"comm","notes":"at least 100 words","prompt":fffcregulartheprompt.text.lower(),"url":fffcregularprompt,"challenge":"regular challenge"}) except: pass @@ -314,6 +333,7 @@ try: ficlettheprompt = ficletprompttext.find("a") print("ficlet-zone (any): \033[1m" + ficlettheprompt.text.lower() + "\033[0m (" + ficletprompt + ")\n") thefile.write("- [[" + ficletprompt + "][ficlet-zone]] (any): *" + ficlettheprompt.text.lower() + "*\n") + prompts.append({"source":"ficlet-zone","type":"comm","notes":"any length","prompt":ficlettheprompt.text.lower(),"url":ficletprompt}) except: pass @@ -340,6 +360,7 @@ try: hourlypromptthishour = str(hourlypromptmedian[0])[5:-5] print("hourlyprompts (any): \033[1m" + hourlypromptthishour.lower() + "\033[0m (" + hourlyprompt + ")\n") thefile.write("- [[" + hourlyprompt + "][hourlyprompts]] (any): *" + hourlypromptthishour.lower() + "*\n") + prompts.append({"source":"hourlyprompts","type":"comm","notes":"any length","prompt":hourlypromptthishour.lower(),"url":hourlyprompt}) except: pass @@ -381,6 +402,7 @@ if themonth != 4 and themonth != 8 and themonth != 12: ssbingofinal = "; ".join(ssbingoclean).lower() print("sweet and short bingo (up to 500 words, separate or combined): \033[1m" + ssbingofinal + "\033[0m (" + ssbingoprompt + ")\n") thefile.write("- [[" + ssbingoprompt + "][sweet and short bingo]] (up to 500 words, separate or combined): *" + ssbingofinal + "*\n") + prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 500 words, separate or combined","prompt":ssbingoclean,"url":ssbingoprompt,"challenge":"bingo"}) except: pass @@ -398,18 +420,18 @@ if themonth != 4 and themonth != 8 and themonth != 12: ssquickypromptnew = (ssquickyurl["href"]) ssquickypromptpage = requests.get(ssquickypromptnew) ssquickypromptsoup = BeautifulSoup(ssquickypromptpage.content, "html.parser") - promptcatch = ".*New Prompts Here" - # ssquickytheprompt = ssquickypromptsoup.find_all("h4",string = re.compile(promptcatch)) ssquickytheprompt = ssquickypromptsoup.find_all(class_="comment") ssquickycomments = [] for comment in ssquickytheprompt: - if re.search("New Prompts Here",str(comment)): + if re.search("new prompts here",str(comment),re.IGNORECASE): commenttext = re.findall(r"
",str(comment)) - commentprompt = re.sub("<.*?>","",str(commenttext)) + commentprompt = re.sub("<.*?>","",str(commenttext)).replace("\\'","'") ssquickycomments.append(str(commentprompt)[2:-2]) + ssquickycomments = ssquickycomments[1:] ssquickycprompt = "; ".join(ssquickycomments) print("sweet and short comment quicky (up to 100 words): \033[1m" + ssquickycprompt.lower() + "\033[0m (" + ssquickyprompt + ")\n") thefile.write("- [[" + ssquickyprompt + "][sweet and short comment quicky]] (up to 100 words): *" + ssquickycprompt.lower() + "*\n") + prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 100 words","prompt":ssquickycomments,"url":ssquickyprompt,"challenge":"comment quicky"}) elif alternate == "picture": sspicture = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+picture+prompt+fun?mode=and&style=light&tag=%21new+challenge,challenge:+picture+prompt+fun&mode=and" sspicturepage = requests.get(sspicture) @@ -425,6 +447,7 @@ if themonth != 4 and themonth != 8 and themonth != 12: sspictureprompttext = sspicturepromptsoup.find("h3") print("sweet and short picture prompts (up to 300 words): \033[1m" + sspictureprompttext.text.lower() + "\033[0m (" + sspictureprompt + ")\n") thefile.write("- [[" + sspictureprompt + "][sweet and short picture prompts]] (up to 300 words): *" + sspictureprompttext.text.lower() + "*\n") + prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 300 words","prompt":sspictureprompttext.text.lower(),"url":sspictureprompts,"challenge":"picture prompt fun"}) except: pass @@ -447,6 +470,7 @@ if themonth != 4 and themonth != 8 and themonth != 12: ssmonthlypromptfinal = str(ssmonthlypromptstripthree)[2:-2] print("sweet and short monthly prompts (up to 500 words based on at least 10 prompts): \033[1m" + ssmonthlypromptfinal + "\033[0m (" + ssmonthlyprompt + ")\n") thefile.write("- [[" + ssmonthlyprompt + "][sweet and short monthly prompts]] (up to 500 words based on at least 10 prompts): *" + ssmonthlypromptfinal + "*\n") + prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 500 words based on at least 10 prompts","prompt":ssmonthlypromptfinal,"url":ssmonthlyprompt,"challenge":"monthly prompt"}) except: pass @@ -468,7 +492,7 @@ if themonth != 4 and themonth != 8 and themonth != 12: ssonepromptfinal = re.sub("2. ","; ",ssonepromptstriptwo) print("sweet and short one sentence (up to 500 words, use one or both lines as the start and/or end): \033[1m" + ssonepromptfinal + "\033[0m (" + ssoneprompt + ")\n") thefile.write("- [[" + ssoneprompt + "][sweet and short one sentence]] (up to 500 words, use one or both lines as the start and/or end): *" + ssonepromptfinal + "*\n") - + prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 500 words, use one or both lines as the start and/or end","prompt":ssonepromptfinal,"url":ssoneprompt,"challenge":"only one"}) except: pass @@ -487,5 +511,30 @@ try: vocabtheprompt = vocabprompttext.find("strong") print("vocab-drabbles (50–500 words): \033[1m" + vocabtheprompt.text.lower() + "\033[0m (" + vocabprompt + ")\n") thefile.write("- [[" + vocabprompt + "][vocab-drabbles]] (50–500 words): *" + vocabtheprompt.text.lower() + "*\n") + prompts.append({"source":"vocab-drabbles","type":"comm","notes":"50–500 words","prompt":vocabtheprompt.text.lower(),"url":vocabprompt}) except: pass + + +if os.path.exists("prompts.html"): + os.remove("prompts.html") + +htmlfile = open("prompts.html", "a") +htmlfile.write("\n\n \n \n \n \n \n \n \n \n Multifandom prompt communities on DW\n \n \n

Multifandom prompt communities on DW

\n
\n") +for prompt in prompts: + htmlfile.write("

\"[community" + prompt["source"].replace("-","_") + "") + try: + if prompt["challenge"]: + htmlfile.write(" " + prompt["challenge"]) + except: + pass + if type(prompt["prompt"]) == list: + htmlfile.write(": prompt post (" + prompt["notes"] + ")

\n
    \n") + for theprompt in prompt["prompt"]: + htmlfile.write("
  • " + theprompt + "
    • \n") + htmlfile.write("
") + else: + htmlfile.write(": " + prompt["prompt"] + " (" + prompt["notes"] + ")

") + htmlfile.write("
\n") +htmlfile.write("
\n

Generated " + datetime.now().strftime("%H:%M (UK time), %-d %B %Y") + "

\n \n") +htmlfile.close()