Update prompt scraper to build an HTML page with the prompts

.gitignore

@@ -1,3 +1,3 @@
-ficheader.html
+prompts.html
 prompts.org
 __pycache__/*

prompts.css

@@ -0,0 +1,47 @@
+html {
+  text-transform: lowercase;
+  color: #808080;
+  background-color: #d5d5ef;
+  /* opacity: 0.8; */
+  background-image:  repeating-radial-gradient( circle at 0 0, transparent 0, #d5d5ef 28px ), repeating-linear-gradient( #cfcfe755, #cfcfe7 );
+  font-family: "Poppins", sans-serif;
+}
+
+a {
+  color: #999999;
+  text-decoration: none;
+}
+
+a:hover {
+  text-decoration: underline;
+}
+
+body {
+  margin: 0 auto;
+  max-width: 800px;
+}
+
+div.promptwrapper > div.promptcomm {
+  border: 1px solid #808080;
+  padding: 0 5px;
+}
+
+div.promptcomm:first-of-type {
+  border-radius: 5px 5px 0 0;
+}
+
+div.promptcomm:last-of-type {
+  border-radius: 0 0 5px 5px;
+}
+
+div.timestamp {
+  text-align: right;
+}
+
+.prompt {
+  font-weight: 700;
+}
+
+.notes {
+  color: #b3b3b3;
+}

promptscrape.py

@@ -18,6 +18,8 @@ today = int(date.today().strftime("%d"))
 month = str(date.today().strftime("%B"))
 monthstring = ".*" + month + ".*"
 
+prompts = []
+
 try:
     cent = "https://100words.dreamwidth.org/tag/!prompt?style=light&tag=%21prompt"
     centpage = requests.get(cent)
@@ -33,6 +35,7 @@ try:
     centtheprompt = centprompttext.find("strong")
     print("100words (100 words): \033[1m" + centtheprompt.text.lower() + "\033[0m (" + centprompt + ")\n")
     thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centtheprompt.text.lower() + "*\n")
+    prompts.append({"source":"100words","type":"comm","notes":"100 words","prompt":centtheprompt.text.lower(),"url":centprompt})
 except:
     pass
 
@@ -88,6 +91,7 @@ try:
         adstrippable = adstrippable[1:]
     print("anythingdrabble (100, 200, 300, 400, or 500 words): \033[1m" + adstrippable.lower() + "\033[0m (" + adprompt + ")\n")
     thefile.write("- [[" + adprompt + "][anythingdrabble]] (100, 200, 300, 400, or 500 words): *" + adstrippable.lower() + "*\n")
+    prompts.append({"source":"anythingdrabble","type":"comm","notes":"100, 200, 300, 400, or 500 words","prompt":adstrippable.lower(),"url":adprompt})
 except:
     pass
 
@@ -106,6 +110,7 @@ try:
     dovetheprompt = doveprompttext.find("i")
     print("dove-drabbles (any): \033[1m" + dovetheprompt.text.lower() + "\033[0m (" + doveprompt + ")\n")
     thefile.write("- [[" + doveprompt + "][dove-drabbles]] (any): *" + dovetheprompt.text.lower() + "*\n")
+    prompts.append({"source":"dove-drabbles","type":"comm","notes":"any length","prompt":dovetheprompt.text.lower(),"url":doveprompt})
 except:
     pass
 
@@ -126,7 +131,13 @@ try:
         zonetheprompt = zoneprompttext.find("strong")
         print("drabble-zone (100 or 200 words): \033[1m" + zonetheprompt.text.lower() + "\033[0m (" + zoneprompt + ")\n")
         thefile.write("- [[" + zoneprompt + "][drabble-zone]] (100 or 200 words): *" + zonetheprompt.text.lower() + "*\n")
-        
+        prompts.append({"source":"drabble-zone","type":"comm","notes":"100 or 200 words","prompt":zonetheprompt.text.lower(),"url":zoneprompt})
+except:
+    pass
+
+try:
+    with requests.Session() as s:
+        response = s.post(login_url , data)
         emotion = "https://emotion100.dreamwidth.org/tag/*modpost?style=light&tag=%2Amodpost"
         emotionpage = s.get(emotion)
         emotionsoup = BeautifulSoup(emotionpage.content, "html.parser")
@@ -141,6 +152,7 @@ try:
         emotiontheprompt = emotionprompttext.find_all("span")[-1]
         print("emotion100 (100 words or a multiple of 100): \033[1m" + emotiontheprompt.text.lower() + "\033[0m (" + emotionprompt + ")\n")
         thefile.write("- [[" + emotionprompt + "][emotion100]] (100 words or a multiple of 100): *" + emotiontheprompt.text.lower() + "*\n")
+        prompts.append({"source":"emotion100","type":"comm","notes":"100 words or a multiple of 100","prompt":emotiontheprompt.text.lower(),"url":emotionprompt})
 except:
     pass
 
@@ -184,6 +196,7 @@ try:
                 ffaformat = "; ".join(ffacentnew)
         print("fail-fandomanon (any): \033[1m" + ffaformat.lower() + "\033[0m (" + ffaprompt + ")\n")
         thefile.write("- [[" + ffaprompt + "][fail-fandomanon]] (any): *" + ffaformat.lower() + "*\n")
+        prompts.append({"source":"fail-fandomanon","type":"comm","notes":"any length","prompt":ffacentnew,"url":ffaprompt})
 except:
     pass
 
@@ -206,6 +219,7 @@ try:
         fandomtheprompt = fandomprompttext.find("td")
         print("fandomweekly (any, competitive): \033[1m" + fandomtheprompt.text.lower() + "\033[0m (" + fandomprompt + ")\n")
         thefile.write("- [[" + fandomprompt + "][fandomweekly]] (any, competitive): *" + fandomtheprompt.text.lower() + "*\n")
+        prompts.append({"source":"fandomweekly","type":"comm","notes":"any length, competitive","prompt":fandomtheprompt.text.lower(),"url":fandomprompt})
 except:
     pass
 
@@ -224,6 +238,7 @@ try:
     flashtheprompt = flashprompttext.find("center")
     print("fan-flashworks (any, can’t post elsewhere until round is closed): \033[1m" + flashtheprompt.text.lower() + "\033[0m (" + flashprompt + ")\n")
     thefile.write("- [[" + flashprompt + "][fan-flashworks]] (any, can’t post elsewhere until round is closed): *" + flashtheprompt.text.lower() + "*\n")
+    prompts.append({"source":"fan-flashworks","type":"comm","notes":"any length, can’t post elsewhere until round is closed","prompt":flashtheprompt.text.lower(),"url":flashprompt})
 except:
     pass
 
@@ -264,6 +279,7 @@ try:
             fffclittleprompttext = fffclittlepromptsoup.find("h3")
             print("fffc little special (at least 100 words): \033[1m" + fffclittleprompttext.text.lower() + "\033[0m (" + fffclittleprompt + ")\n")
             thefile.write("- [[" + fffclittleprompt + "][fffc little special]] (at least 100 words): *" + fffclittleprompttext.text.lower() + "*\n")
+            prompts.append({"source":"fffc","type":"comm","notes":"at least 100 words","prompt":fffclittleprompttext.text.lower(),"url":fffclittleprompt,"challenge":"little special"})
         fffcmadnessprompts = fffcsoup.find_all("h3", string=lambda text: "froday madness" in text.lower())
         fffcmadnesssubsoup = BeautifulSoup(str(fffcmadnessprompts[0]), "html.parser")
         fffcmadnessurl = fffcmadnesssubsoup.find("a")
@@ -275,6 +291,7 @@ try:
         fffcmadnesstheprompt = fffcmadnessprompttext.find("b")
         print("fffc madness (at least 2000 words): \033[1m" + fffcmadnesstheprompt.text.lower() + "\033[0m (" + fffcmadnessprompt + ")\n")
         thefile.write("- [[" + fffcmadnessprompt + "][fffc madness]] (at least 2000 words): *" + fffcmadnesstheprompt.text.lower() + "*\n")
+        prompts.append({"source":"fffc","type":"comm","notes":"at least 2000 words","prompt":fffcmadnesstheprompt.text.lower(),"url":fffcmadnessprompt,"challenge":"froday madness"})
         fffcmonthlyprompts = fffcsoup.find_all("h3", string=re.compile(monthstring))
         fffcmonthlysubsoup = BeautifulSoup(str(fffcmonthlyprompts[0]), "html.parser")
         fffcmonthlyurl = fffcmonthlysubsoup.find("a")
@@ -285,6 +302,7 @@ try:
         fffcmonthlyprompttext = fffcmonthlypromptsoup.find("h3")
         print("fffc monthly special (usually at least 500 words): \033[1m" + fffcmonthlyprompttext.text.lower() + "\033[0m (" + fffcmonthlyprompt + ")\n")
         thefile.write("- [[" + fffcmonthlyprompt + "][fffc monthly special]] (usually at least 500 words): *" + fffcmonthlyprompttext.text.lower() + "*\n")
+        prompts.append({"source":"fffc","type":"comm","notes":"usually at least 500 words","prompt":fffcmonthlyprompttext.text.lower(),"url":fffcmonthlyprompt,"challenge":"monthly challenge"})
         fffcregularprompts = fffcsoup.find_all("h3", string=lambda text: "regular challenge" in text.lower())
         fffcregularsubsoup = BeautifulSoup(str(fffcregularprompts[0]), "html.parser")
         fffcregularurl = fffcregularsubsoup.find("a")
@@ -296,6 +314,7 @@ try:
         fffcregulartheprompt = fffcregularprompttext.find("b")
         print("fffc regular challenge (at least 100 words): \033[1m" + fffcregulartheprompt.text.lower() + "\033[0m (" + fffcregularprompt + ")\n")
         thefile.write("- [[" + fffcregularprompt + "][fffc regular challenge]] (at least 100 words): *" + fffcregulartheprompt.text.lower() + "*\n")
+        prompts.append({"source":"fffc","type":"comm","notes":"at least 100 words","prompt":fffcregulartheprompt.text.lower(),"url":fffcregularprompt,"challenge":"regular challenge"})
 except:
     pass
 
@@ -314,6 +333,7 @@ try:
     ficlettheprompt = ficletprompttext.find("a")
     print("ficlet-zone (any): \033[1m" + ficlettheprompt.text.lower() + "\033[0m (" + ficletprompt + ")\n")
     thefile.write("- [[" + ficletprompt + "][ficlet-zone]] (any): *" + ficlettheprompt.text.lower() + "*\n")
+    prompts.append({"source":"ficlet-zone","type":"comm","notes":"any length","prompt":ficlettheprompt.text.lower(),"url":ficletprompt})
 except:
     pass
 
@@ -340,6 +360,7 @@ try:
         hourlypromptthishour = str(hourlypromptmedian[0])[5:-5]
         print("hourlyprompts (any): \033[1m" + hourlypromptthishour.lower() + "\033[0m (" + hourlyprompt + ")\n")
         thefile.write("- [[" + hourlyprompt + "][hourlyprompts]] (any): *" + hourlypromptthishour.lower() + "*\n")
+        prompts.append({"source":"hourlyprompts","type":"comm","notes":"any length","prompt":hourlypromptthishour.lower(),"url":hourlyprompt})
 except:
     pass
 
@@ -381,6 +402,7 @@ if themonth != 4 and themonth != 8 and themonth != 12:
                 ssbingofinal = "; ".join(ssbingoclean).lower()
             print("sweet and short bingo (up to 500 words, separate or combined): \033[1m" + ssbingofinal + "\033[0m (" + ssbingoprompt + ")\n")
             thefile.write("- [[" + ssbingoprompt + "][sweet and short bingo]] (up to 500 words, separate or combined): *" + ssbingofinal + "*\n")
+            prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 500 words, separate or combined","prompt":ssbingoclean,"url":ssbingoprompt,"challenge":"bingo"})
     except:
         pass
 
@@ -398,18 +420,18 @@ if themonth != 4 and themonth != 8 and themonth != 12:
                 ssquickypromptnew = (ssquickyurl["href"])
                 ssquickypromptpage = requests.get(ssquickypromptnew)
                 ssquickypromptsoup = BeautifulSoup(ssquickypromptpage.content, "html.parser")
-                promptcatch = ".*New Prompts Here"
-                # ssquickytheprompt = ssquickypromptsoup.find_all("h4",string = re.compile(promptcatch))
                 ssquickytheprompt = ssquickypromptsoup.find_all(class_="comment")
                 ssquickycomments = []
                 for comment in ssquickytheprompt:
-                    if re.search("New Prompts Here",str(comment)):
+                    if re.search("new prompts here",str(comment),re.IGNORECASE):
                         commenttext = re.findall(r"<div class=\"comment-content\".*?</div>",str(comment))
-                        commentprompt = re.sub("<.*?>","",str(commenttext))
+                        commentprompt = re.sub("<.*?>","",str(commenttext)).replace("\\'","'")
                         ssquickycomments.append(str(commentprompt)[2:-2])
+                ssquickycomments = ssquickycomments[1:]
                 ssquickycprompt = "; ".join(ssquickycomments)
                 print("sweet and short comment quicky (up to 100 words): \033[1m" + ssquickycprompt.lower() + "\033[0m (" + ssquickyprompt + ")\n")
                 thefile.write("- [[" + ssquickyprompt + "][sweet and short comment quicky]] (up to 100 words): *" + ssquickycprompt.lower() + "*\n")
+                prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 100 words","prompt":ssquickycomments,"url":ssquickyprompt,"challenge":"comment quicky"})
             elif alternate == "picture":
                 sspicture = "https://sweetandshort.dreamwidth.org/tag/!new+challenge,challenge:+picture+prompt+fun?mode=and&style=light&tag=%21new+challenge,challenge:+picture+prompt+fun&mode=and"
                 sspicturepage = requests.get(sspicture)
@@ -425,6 +447,7 @@ if themonth != 4 and themonth != 8 and themonth != 12:
                 sspictureprompttext = sspicturepromptsoup.find("h3")
                 print("sweet and short picture prompts (up to 300 words): \033[1m" + sspictureprompttext.text.lower() + "\033[0m (" + sspictureprompt + ")\n")
                 thefile.write("- [[" + sspictureprompt + "][sweet and short picture prompts]] (up to 300 words): *" + sspictureprompttext.text.lower() + "*\n")
+                prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 300 words","prompt":sspictureprompttext.text.lower(),"url":sspictureprompts,"challenge":"picture prompt fun"})
     except:
         pass
 
@@ -447,6 +470,7 @@ if themonth != 4 and themonth != 8 and themonth != 12:
         ssmonthlypromptfinal = str(ssmonthlypromptstripthree)[2:-2]
         print("sweet and short monthly prompts (up to 500 words based on at least 10 prompts): \033[1m" + ssmonthlypromptfinal + "\033[0m (" + ssmonthlyprompt + ")\n")
         thefile.write("- [[" + ssmonthlyprompt + "][sweet and short monthly prompts]] (up to 500 words based on at least 10 prompts): *" + ssmonthlypromptfinal + "*\n")
+        prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 500 words based on at least 10 prompts","prompt":ssmonthlypromptfinal,"url":ssmonthlyprompt,"challenge":"monthly prompt"})
     except:
         pass
 
@@ -468,7 +492,7 @@ if themonth != 4 and themonth != 8 and themonth != 12:
             ssonepromptfinal = re.sub("2. ","; ",ssonepromptstriptwo)
             print("sweet and short one sentence (up to 500 words, use one or both lines as the start and/or end): \033[1m" + ssonepromptfinal + "\033[0m (" + ssoneprompt + ")\n")
             thefile.write("- [[" + ssoneprompt + "][sweet and short one sentence]] (up to 500 words, use one or both lines as the start and/or end): *" + ssonepromptfinal + "*\n")
-
+            prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 500 words, use one or both lines as the start and/or end","prompt":ssonepromptfinal,"url":ssoneprompt,"challenge":"only one"})
     except:
         pass
 
@@ -487,5 +511,30 @@ try:
     vocabtheprompt = vocabprompttext.find("strong")
     print("vocab-drabbles (50–500 words): \033[1m" + vocabtheprompt.text.lower() + "\033[0m (" + vocabprompt + ")\n")
     thefile.write("- [[" + vocabprompt + "][vocab-drabbles]] (50–500 words): *" + vocabtheprompt.text.lower() + "*\n")
+    prompts.append({"source":"vocab-drabbles","type":"comm","notes":"50–500 words","prompt":vocabtheprompt.text.lower(),"url":vocabprompt})
 except:
     pass
+
+
+if os.path.exists("prompts.html"):
+    os.remove("prompts.html")
+
+htmlfile = open("prompts.html", "a")
+htmlfile.write("<!DOCTYPE html>\n<html lang=\"en\">\n  <head>\n    <meta charset=\"utf-8\">\n    <meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">\n    <link rel=\"stylesheet\" href=\"prompts.css\">\n    <link rel=\"preconnect\" href=\"https://fonts.googleapis.com\">\n    <link rel=\"preconnect\" href=\"https://fonts.gstatic.com\" crossorigin>\n    <link href=\"https://fonts.googleapis.com/css2?family=Poppins:wght@400;700&display=swap\" rel=\"stylesheet\">\n    <meta name=\"theme-color\" content=\"#d5d5ef\">\n    <title>Multifandom prompt communities on DW</title>\n  </head>\n  <body>\n    <h1>Multifandom prompt communities on DW</h1>\n    <div class=\"promptwrapper\">\n")
+for prompt in prompts:
+    htmlfile.write("      <div class=\"promptcomm\"><p><span style=\"white-space: nowrap;\" class=\"ljuser\"><a href=\"https://" + prompt["source"] + ".dreamwidth.org/profile\"><img src=\"https://www.dreamwidth.org/img/silk/identity/community.png\" alt=\"[community profile]\" width=\"17\" height=\"17\" style=\"border: 0; padding-right: 1px;\" /></a><a href=\"https://" + prompt["source"] + ".dreamwidth.org/\"><b>" + prompt["source"].replace("-","_") + "</b></a></span>")
+    try:
+        if prompt["challenge"]:
+            htmlfile.write(" " + prompt["challenge"])
+    except:
+        pass
+    if type(prompt["prompt"]) == list:
+        htmlfile.write(": <a href=\"" + prompt["url"] + "\">prompt post</a> <span class=\"notes\">(" + prompt["notes"] + ")</span></p>\n      <ul>\n")
+        for theprompt in prompt["prompt"]:
+            htmlfile.write("        <li><span class=\"prompt\">" + theprompt + "</span></li>\n")
+        htmlfile.write("      </ul>")
+    else:
+        htmlfile.write(": <a href=\"" + prompt["url"] + "\"><span class=\"prompt\">" + prompt["prompt"] + "</span></a> <span class=\"notes\">(" + prompt["notes"] + ")</span></p>")
+    htmlfile.write("</div>\n")
+htmlfile.write("    </div>\n    <div class=\"timestamp\"><p>Generated " + datetime.now().strftime("%H:%M (UK time), %-d %B %Y") + "</p></div>\n  </body>\n</html>")
+htmlfile.close()