diff --git a/.gitignore b/.gitignore index 7d6d9e7..5abfae0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ -ficheader.html +albums.py +prompts.html prompts.org __pycache__/* \ No newline at end of file diff --git a/README.org b/README.org index c11e52b..d5d7809 100644 --- a/README.org +++ b/README.org @@ -1,6 +1,7 @@ * Python utilities for personal and fannish purposes -- =ao3scrape.py= – downloads all an author’s works from AO3 (notes [[https://tobli.dreamwidth.org/45337.html][here]]) +- =ao3scrape.py= – downloads all an author’s works from AO3 - =cellopractice.py= – selects some exercises from Walter Mengler’s /Fit in 15 Minutes/, according to the author’s recommendations +- =ffyuletide.py= – checks which Final Fantasy fandoms are Yuletide-eligible - =peterson.py= – questions everything - =promptscrape.py= – scrapes Dreamwidth writing challenge communities for prompts - =randomline.py= – returns a random line from a file diff --git a/ffyuletide.py b/ffyuletide.py new file mode 100644 index 0000000..c94a04a --- /dev/null +++ b/ffyuletide.py @@ -0,0 +1,62 @@ +import re, requests, time +from bs4 import BeautifulSoup + +# grab subtags of "final fantasy series" + +series = "https://archiveofourown.org/tags/Final%20Fantasy%20Series" +seriespage = requests.get(series) +seriessoup = BeautifulSoup(seriespage.content,"html.parser") +fandoms = seriessoup.find(class_="sub").find("ul",{ "class" : "tree" }).findAll("li", recursive=False) + +# open search page with filters applied + +filterpage = "https://archiveofourown.org/works?work_search%5Bsort_column%5D=revised_at&work_search%5Bother_tag_names%5D=&work_search%5Bexcluded_tag_names%5D=&work_search%5Bcrossover%5D=&work_search%5Bcomplete%5D=T&work_search%5Bwords_from%5D=1000&work_search%5Bwords_to%5D=&work_search%5Bdate_from%5D=&work_search%5Bdate_to%5D=&work_search%5Bquery%5D=&work_search%5Blanguage_id%5D=en&commit=Sort+and+Filter&tag_id=" +eligible = [] +ineligible = [] +with requests.Session() as s: + loginpage = s.get("https://archiveofourown.org/users/login") + loginsoup = BeautifulSoup(loginpage.content,"html.parser") + token = (loginsoup.find('form', class_='new_user') + .find('input', attrs={'name': 'authenticity_token'}) + .get('value')) + payload = {'user[login]': "translinkni", + 'user[password]': "fakeapi", + 'user[remember_me]': '1', + 'authenticity_token': token + } + post = s.post("https://archiveofourown.org/users/login",data=payload) + for fandom in fandoms: + time.sleep(3) + tagurl = (str(fandom.find("a")["href"]))[6:] + tagname = str(fandom.find(class_="tag").text) + print("Checking " + tagname + " (" + str(fandoms.index(fandom) + 1) + " of " + str(len(fandoms)) + ")") + tagpage = s.get(filterpage + tagurl) + tagsoup = BeautifulSoup(tagpage.content,"html.parser") + + # get the number + + heading = (tagsoup.find("h2",class_="heading")).text + stripone = re.sub("\n","",heading) + striptwo = re.sub(" Work.*","",stripone) + stripthree = re.sub(".* of ","",striptwo) + stripfour = re.sub(",","",stripthree) + workcount = int(stripfour) + + # if number < 1000, add to eligible list + + fandomdict = {"name":tagname,"count":workcount} + + if workcount < 1000: + eligible.append(fandomdict) + else: + ineligible.append(fandomdict) + +eligible = sorted(eligible,key=lambda d: d["count"],reverse=True) +ineligible = sorted(ineligible,key=lambda d: d["count"],reverse=True) + +print("\nCurrently eligible:\n") +for fandom in eligible: + print(fandom["name"] + " (" + str(fandom["count"]) + ")") +print("\nIneligible:\n") +for fandom in ineligible: + print(fandom["name"] + " (" + str(fandom["count"]) + ")") diff --git a/otd.py b/otd.py new file mode 100644 index 0000000..713d2d2 --- /dev/null +++ b/otd.py @@ -0,0 +1,19 @@ +import datetime,os + +year = int(datetime.datetime.strftime(datetime.datetime.today(),"%Y")) +monthstring = datetime.datetime.strftime(datetime.datetime.today(),"%m") +datestring = datetime.datetime.strftime(datetime.datetime.today(),"%d") + +fileplace = "/home/mdd/Documents/drive/org/journal/" + +thisyear = year + +while year > 1993: + year -= 1 + target = fileplace + str(year) + "/" + monthstring + "/" + str(year) + "-" + monthstring + "-" + datestring + ".org" + if os.path.exists(target): + if year < thisyear -1: + print("Press enter to continue") + input() + print(open(target,"r").read()) + diff --git a/prompts.css b/prompts.css new file mode 100644 index 0000000..6f49399 --- /dev/null +++ b/prompts.css @@ -0,0 +1,47 @@ +html { + text-transform: lowercase; + color: #808080; + background-color: #d5d5ef; + /* opacity: 0.8; */ + background-image: repeating-radial-gradient( circle at 0 0, transparent 0, #d5d5ef 28px ), repeating-linear-gradient( #cfcfe5, #cfcfe7 ); + font-family: "Poppins", sans-serif; +} + +a { + color: #999999; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +body { + margin: 0 auto; + max-width: 800px; +} + +div.promptwrapper > div.promptcomm { + border: 1px solid #808080; + padding: 0 5px; +} + +div.promptcomm:first-of-type { + border-radius: 5px 5px 0 0; +} + +div.promptcomm:last-of-type { + border-radius: 0 0 5px 5px; +} + +div.timestamp { + text-align: right; +} + +.prompt { + font-weight: 700; +} + +.notes { + color: #b3b3b3; +} diff --git a/promptscrape.py b/promptscrape.py index b4000d6..047b255 100644 --- a/promptscrape.py +++ b/promptscrape.py @@ -18,6 +18,8 @@ today = int(date.today().strftime("%d")) month = str(date.today().strftime("%B")) monthstring = ".*" + month + ".*" +prompts = [] + try: cent = "https://100words.dreamwidth.org/tag/!prompt?style=light&tag=%21prompt" centpage = requests.get(cent) @@ -33,6 +35,7 @@ try: centtheprompt = centprompttext.find("strong") print("100words (100 words): \033[1m" + centtheprompt.text.lower() + "\033[0m (" + centprompt + ")\n") thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centtheprompt.text.lower() + "*\n") + prompts.append({"source":"100words","type":"comm","notes":"100 words","prompt":centtheprompt.text.lower(),"url":centprompt}) except: pass @@ -88,6 +91,7 @@ try: adstrippable = adstrippable[1:] print("anythingdrabble (100, 200, 300, 400, or 500 words): \033[1m" + adstrippable.lower() + "\033[0m (" + adprompt + ")\n") thefile.write("- [[" + adprompt + "][anythingdrabble]] (100, 200, 300, 400, or 500 words): *" + adstrippable.lower() + "*\n") + prompts.append({"source":"anythingdrabble","type":"comm","notes":"100, 200, 300, 400, or 500 words","prompt":adstrippable.lower(),"url":adprompt}) except: pass @@ -95,7 +99,7 @@ try: dove = "https://dove-drabbles.dreamwidth.org/?style=light" dovepage = requests.get(dove) dovesoup = BeautifulSoup(dovepage.content, "html.parser") - doveprompts = dovesoup.find_all("h3", string=lambda text: "prompt post" in text.lower()) + doveprompts = dovesoup.find_all("h3", string=lambda text: "prompt" in text.lower()) dovesubsoup = BeautifulSoup(str(doveprompts[0]), "html.parser") doveurl = dovesubsoup.find("a") doveprompt = (doveurl["href"]) @@ -106,6 +110,7 @@ try: dovetheprompt = doveprompttext.find("i") print("dove-drabbles (any): \033[1m" + dovetheprompt.text.lower() + "\033[0m (" + doveprompt + ")\n") thefile.write("- [[" + doveprompt + "][dove-drabbles]] (any): *" + dovetheprompt.text.lower() + "*\n") + prompts.append({"source":"dove-drabbles","type":"comm","notes":"any length","prompt":dovetheprompt.text.lower(),"url":doveprompt}) except: pass @@ -123,10 +128,16 @@ try: zonepromptpage = s.get(zonepromptnew) zonepromptsoup = BeautifulSoup(zonepromptpage.content, "html.parser") zoneprompttext = zonepromptsoup.find(class_="entry-content") - zonetheprompt = zoneprompttext.find("strong") + zonetheprompt = zoneprompttext.find("div") print("drabble-zone (100 or 200 words): \033[1m" + zonetheprompt.text.lower() + "\033[0m (" + zoneprompt + ")\n") thefile.write("- [[" + zoneprompt + "][drabble-zone]] (100 or 200 words): *" + zonetheprompt.text.lower() + "*\n") - + prompts.append({"source":"drabble-zone","type":"comm","notes":"100 or 200 words","prompt":zonetheprompt.text.lower(),"url":zoneprompt}) +except: + pass + +try: + with requests.Session() as s: + response = s.post(login_url , data) emotion = "https://emotion100.dreamwidth.org/tag/*modpost?style=light&tag=%2Amodpost" emotionpage = s.get(emotion) emotionsoup = BeautifulSoup(emotionpage.content, "html.parser") @@ -141,6 +152,7 @@ try: emotiontheprompt = emotionprompttext.find_all("span")[-1] print("emotion100 (100 words or a multiple of 100): \033[1m" + emotiontheprompt.text.lower() + "\033[0m (" + emotionprompt + ")\n") thefile.write("- [[" + emotionprompt + "][emotion100]] (100 words or a multiple of 100): *" + emotiontheprompt.text.lower() + "*\n") + prompts.append({"source":"emotion100","type":"comm","notes":"100 words or a multiple of 100","prompt":emotiontheprompt.text.lower(),"url":emotionprompt}) except: pass @@ -184,6 +196,7 @@ try: ffaformat = "; ".join(ffacentnew) print("fail-fandomanon (any): \033[1m" + ffaformat.lower() + "\033[0m (" + ffaprompt + ")\n") thefile.write("- [[" + ffaprompt + "][fail-fandomanon]] (any): *" + ffaformat.lower() + "*\n") + prompts.append({"source":"fail-fandomanon","type":"comm","notes":"any length","prompt":ffacentnew,"url":ffaprompt}) except: pass @@ -206,6 +219,7 @@ try: fandomtheprompt = fandomprompttext.find("td") print("fandomweekly (any, competitive): \033[1m" + fandomtheprompt.text.lower() + "\033[0m (" + fandomprompt + ")\n") thefile.write("- [[" + fandomprompt + "][fandomweekly]] (any, competitive): *" + fandomtheprompt.text.lower() + "*\n") + prompts.append({"source":"fandomweekly","type":"comm","notes":"any length, competitive","prompt":fandomtheprompt.text.lower(),"url":fandomprompt}) except: pass @@ -224,6 +238,28 @@ try: flashtheprompt = flashprompttext.find("center") print("fan-flashworks (any, can’t post elsewhere until round is closed): \033[1m" + flashtheprompt.text.lower() + "\033[0m (" + flashprompt + ")\n") thefile.write("- [[" + flashprompt + "][fan-flashworks]] (any, can’t post elsewhere until round is closed): *" + flashtheprompt.text.lower() + "*\n") + prompts.append({"source":"fan-flashworks","type":"comm","notes":"any length, can’t post elsewhere until round is closed","prompt":flashtheprompt.text.lower(),"url":flashprompt}) +except: + pass + +try: + with requests.Session() as s: + response = s.post(login_url , data) + fsf = "https://femslashfete.dreamwidth.org/tag/admin?style=light&tag=admin" + fsfpage = s.get(fsf) + fsfsoup = BeautifulSoup(fsfpage.content, "html.parser") + fsfprompts = fsfsoup.find_all("h3", string=lambda text: "challenge" in text.lower()) + fsfsubsoup = BeautifulSoup(str(fsfprompts[0]), "html.parser") + fsfurl = fsfsubsoup.find("a") + fsfprompt = (fsfurl["href"]) + fsfpromptnew = (fsfurl["href"] + "?style=light") + fsfpromptpage = s.get(fsfpromptnew) + fsfpromptsoup = BeautifulSoup(fsfpromptpage.content, "html.parser") + fsfprompttext = fsfpromptsoup.find(class_="entry-content") + fsftheprompt = fsfprompttext.find("b") + print("femslashfete (at least 100 words, must be femslash): \033[1m" + fsftheprompt.text.lower() + "\033[0m (" + fsfprompt + ")\n") + thefile.write("- [[" + fsfprompt + "][femslashfete]] (at least 100 words, must be femslash): *" + fsftheprompt.text.lower() + "*\n") + prompts.append({"source":"femslashfete","type":"comm","notes":"at least 100 words, must be femslash","prompt":fsftheprompt.text.lower(),"url":fsfprompt}) except: pass @@ -264,6 +300,7 @@ try: fffclittleprompttext = fffclittlepromptsoup.find("h3") print("fffc little special (at least 100 words): \033[1m" + fffclittleprompttext.text.lower() + "\033[0m (" + fffclittleprompt + ")\n") thefile.write("- [[" + fffclittleprompt + "][fffc little special]] (at least 100 words): *" + fffclittleprompttext.text.lower() + "*\n") + prompts.append({"source":"fffc","type":"comm","notes":"at least 100 words","prompt":fffclittleprompttext.text.lower(),"url":fffclittleprompt,"challenge":"little special"}) fffcmadnessprompts = fffcsoup.find_all("h3", string=lambda text: "froday madness" in text.lower()) fffcmadnesssubsoup = BeautifulSoup(str(fffcmadnessprompts[0]), "html.parser") fffcmadnessurl = fffcmadnesssubsoup.find("a") @@ -275,6 +312,7 @@ try: fffcmadnesstheprompt = fffcmadnessprompttext.find("b") print("fffc madness (at least 2000 words): \033[1m" + fffcmadnesstheprompt.text.lower() + "\033[0m (" + fffcmadnessprompt + ")\n") thefile.write("- [[" + fffcmadnessprompt + "][fffc madness]] (at least 2000 words): *" + fffcmadnesstheprompt.text.lower() + "*\n") + prompts.append({"source":"fffc","type":"comm","notes":"at least 2000 words","prompt":fffcmadnesstheprompt.text.lower(),"url":fffcmadnessprompt,"challenge":"froday madness"}) fffcmonthlyprompts = fffcsoup.find_all("h3", string=re.compile(monthstring)) fffcmonthlysubsoup = BeautifulSoup(str(fffcmonthlyprompts[0]), "html.parser") fffcmonthlyurl = fffcmonthlysubsoup.find("a") @@ -285,6 +323,7 @@ try: fffcmonthlyprompttext = fffcmonthlypromptsoup.find("h3") print("fffc monthly special (usually at least 500 words): \033[1m" + fffcmonthlyprompttext.text.lower() + "\033[0m (" + fffcmonthlyprompt + ")\n") thefile.write("- [[" + fffcmonthlyprompt + "][fffc monthly special]] (usually at least 500 words): *" + fffcmonthlyprompttext.text.lower() + "*\n") + prompts.append({"source":"fffc","type":"comm","notes":"usually at least 500 words","prompt":fffcmonthlyprompttext.text.lower(),"url":fffcmonthlyprompt,"challenge":"monthly challenge"}) fffcregularprompts = fffcsoup.find_all("h3", string=lambda text: "regular challenge" in text.lower()) fffcregularsubsoup = BeautifulSoup(str(fffcregularprompts[0]), "html.parser") fffcregularurl = fffcregularsubsoup.find("a") @@ -296,6 +335,7 @@ try: fffcregulartheprompt = fffcregularprompttext.find("b") print("fffc regular challenge (at least 100 words): \033[1m" + fffcregulartheprompt.text.lower() + "\033[0m (" + fffcregularprompt + ")\n") thefile.write("- [[" + fffcregularprompt + "][fffc regular challenge]] (at least 100 words): *" + fffcregulartheprompt.text.lower() + "*\n") + prompts.append({"source":"fffc","type":"comm","notes":"at least 100 words","prompt":fffcregulartheprompt.text.lower(),"url":fffcregularprompt,"challenge":"regular challenge"}) except: pass @@ -314,6 +354,7 @@ try: ficlettheprompt = ficletprompttext.find("a") print("ficlet-zone (any): \033[1m" + ficlettheprompt.text.lower() + "\033[0m (" + ficletprompt + ")\n") thefile.write("- [[" + ficletprompt + "][ficlet-zone]] (any): *" + ficlettheprompt.text.lower() + "*\n") + prompts.append({"source":"ficlet-zone","type":"comm","notes":"any length","prompt":ficlettheprompt.text.lower(),"url":ficletprompt}) except: pass @@ -340,6 +381,7 @@ try: hourlypromptthishour = str(hourlypromptmedian[0])[5:-5] print("hourlyprompts (any): \033[1m" + hourlypromptthishour.lower() + "\033[0m (" + hourlyprompt + ")\n") thefile.write("- [[" + hourlyprompt + "][hourlyprompts]] (any): *" + hourlypromptthishour.lower() + "*\n") + prompts.append({"source":"hourlyprompts","type":"comm","notes":"any length","prompt":hourlypromptthishour.lower(),"url":hourlyprompt}) except: pass @@ -381,6 +423,7 @@ if themonth != 4 and themonth != 8 and themonth != 12: ssbingofinal = "; ".join(ssbingoclean).lower() print("sweet and short bingo (up to 500 words, separate or combined): \033[1m" + ssbingofinal + "\033[0m (" + ssbingoprompt + ")\n") thefile.write("- [[" + ssbingoprompt + "][sweet and short bingo]] (up to 500 words, separate or combined): *" + ssbingofinal + "*\n") + prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 500 words, separate or combined","prompt":ssbingoclean,"url":ssbingoprompt,"challenge":"bingo"}) except: pass @@ -398,18 +441,18 @@ if themonth != 4 and themonth != 8 and themonth != 12: ssquickypromptnew = (ssquickyurl["href"]) ssquickypromptpage = requests.get(ssquickypromptnew) ssquickypromptsoup = BeautifulSoup(ssquickypromptpage.content, "html.parser") - promptcatch = ".*New Prompts Here" - # ssquickytheprompt = ssquickypromptsoup.find_all("h4",string = re.compile(promptcatch)) ssquickytheprompt = ssquickypromptsoup.find_all(class_="comment") ssquickycomments = [] for comment in ssquickytheprompt: - if re.search("New Prompts Here",str(comment)): + if re.search("new prompts here",str(comment),re.IGNORECASE): commenttext = re.findall(r"
Multifandom prompt communities on DW
\n
" + prompt["source"].replace("-","_") + "")
+ try:
+ if prompt["challenge"]:
+ htmlfile.write(" " + prompt["challenge"])
+ except:
+ pass
+ if type(prompt["prompt"]) == list:
+ htmlfile.write(": prompt post (" + prompt["notes"] + ")
\n\n") + for theprompt in prompt["prompt"]: + htmlfile.write("- " + theprompt + "
\n")
+ htmlfile.write("
") + else: + htmlfile.write(": " + prompt["prompt"] + " (" + prompt["notes"] + ")") + htmlfile.write("Generated " + datetime.now().strftime("%H:%M (UK time), %-d %B %Y") + "