diff --git a/.gitignore b/.gitignore index 5abfae0..7d6d9e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -albums.py -prompts.html +ficheader.html prompts.org __pycache__/* \ No newline at end of file diff --git a/README.org b/README.org index d5d7809..c11e52b 100644 --- a/README.org +++ b/README.org @@ -1,7 +1,6 @@ * Python utilities for personal and fannish purposes -- =ao3scrape.py= – downloads all an author’s works from AO3 +- =ao3scrape.py= – downloads all an author’s works from AO3 (notes [[https://tobli.dreamwidth.org/45337.html][here]]) - =cellopractice.py= – selects some exercises from Walter Mengler’s /Fit in 15 Minutes/, according to the author’s recommendations -- =ffyuletide.py= – checks which Final Fantasy fandoms are Yuletide-eligible - =peterson.py= – questions everything - =promptscrape.py= – scrapes Dreamwidth writing challenge communities for prompts - =randomline.py= – returns a random line from a file diff --git a/ffyuletide.py b/ffyuletide.py deleted file mode 100644 index c94a04a..0000000 --- a/ffyuletide.py +++ /dev/null @@ -1,62 +0,0 @@ -import re, requests, time -from bs4 import BeautifulSoup - -# grab subtags of "final fantasy series" - -series = "https://archiveofourown.org/tags/Final%20Fantasy%20Series" -seriespage = requests.get(series) -seriessoup = BeautifulSoup(seriespage.content,"html.parser") -fandoms = seriessoup.find(class_="sub").find("ul",{ "class" : "tree" }).findAll("li", recursive=False) - -# open search page with filters applied - -filterpage = "https://archiveofourown.org/works?work_search%5Bsort_column%5D=revised_at&work_search%5Bother_tag_names%5D=&work_search%5Bexcluded_tag_names%5D=&work_search%5Bcrossover%5D=&work_search%5Bcomplete%5D=T&work_search%5Bwords_from%5D=1000&work_search%5Bwords_to%5D=&work_search%5Bdate_from%5D=&work_search%5Bdate_to%5D=&work_search%5Bquery%5D=&work_search%5Blanguage_id%5D=en&commit=Sort+and+Filter&tag_id=" -eligible = [] -ineligible = [] -with requests.Session() as s: - loginpage = s.get("https://archiveofourown.org/users/login") - loginsoup = BeautifulSoup(loginpage.content,"html.parser") - token = (loginsoup.find('form', class_='new_user') - .find('input', attrs={'name': 'authenticity_token'}) - .get('value')) - payload = {'user[login]': "translinkni", - 'user[password]': "fakeapi", - 'user[remember_me]': '1', - 'authenticity_token': token - } - post = s.post("https://archiveofourown.org/users/login",data=payload) - for fandom in fandoms: - time.sleep(3) - tagurl = (str(fandom.find("a")["href"]))[6:] - tagname = str(fandom.find(class_="tag").text) - print("Checking " + tagname + " (" + str(fandoms.index(fandom) + 1) + " of " + str(len(fandoms)) + ")") - tagpage = s.get(filterpage + tagurl) - tagsoup = BeautifulSoup(tagpage.content,"html.parser") - - # get the number - - heading = (tagsoup.find("h2",class_="heading")).text - stripone = re.sub("\n","",heading) - striptwo = re.sub(" Work.*","",stripone) - stripthree = re.sub(".* of ","",striptwo) - stripfour = re.sub(",","",stripthree) - workcount = int(stripfour) - - # if number < 1000, add to eligible list - - fandomdict = {"name":tagname,"count":workcount} - - if workcount < 1000: - eligible.append(fandomdict) - else: - ineligible.append(fandomdict) - -eligible = sorted(eligible,key=lambda d: d["count"],reverse=True) -ineligible = sorted(ineligible,key=lambda d: d["count"],reverse=True) - -print("\nCurrently eligible:\n") -for fandom in eligible: - print(fandom["name"] + " (" + str(fandom["count"]) + ")") -print("\nIneligible:\n") -for fandom in ineligible: - print(fandom["name"] + " (" + str(fandom["count"]) + ")") diff --git a/otd.py b/otd.py deleted file mode 100644 index 713d2d2..0000000 --- a/otd.py +++ /dev/null @@ -1,19 +0,0 @@ -import datetime,os - -year = int(datetime.datetime.strftime(datetime.datetime.today(),"%Y")) -monthstring = datetime.datetime.strftime(datetime.datetime.today(),"%m") -datestring = datetime.datetime.strftime(datetime.datetime.today(),"%d") - -fileplace = "/home/mdd/Documents/drive/org/journal/" - -thisyear = year - -while year > 1993: - year -= 1 - target = fileplace + str(year) + "/" + monthstring + "/" + str(year) + "-" + monthstring + "-" + datestring + ".org" - if os.path.exists(target): - if year < thisyear -1: - print("Press enter to continue") - input() - print(open(target,"r").read()) - diff --git a/prompts.css b/prompts.css deleted file mode 100644 index 6f49399..0000000 --- a/prompts.css +++ /dev/null @@ -1,47 +0,0 @@ -html { - text-transform: lowercase; - color: #808080; - background-color: #d5d5ef; - /* opacity: 0.8; */ - background-image: repeating-radial-gradient( circle at 0 0, transparent 0, #d5d5ef 28px ), repeating-linear-gradient( #cfcfe5, #cfcfe7 ); - font-family: "Poppins", sans-serif; -} - -a { - color: #999999; - text-decoration: none; -} - -a:hover { - text-decoration: underline; -} - -body { - margin: 0 auto; - max-width: 800px; -} - -div.promptwrapper > div.promptcomm { - border: 1px solid #808080; - padding: 0 5px; -} - -div.promptcomm:first-of-type { - border-radius: 5px 5px 0 0; -} - -div.promptcomm:last-of-type { - border-radius: 0 0 5px 5px; -} - -div.timestamp { - text-align: right; -} - -.prompt { - font-weight: 700; -} - -.notes { - color: #b3b3b3; -} diff --git a/promptscrape.py b/promptscrape.py index 047b255..b4000d6 100644 --- a/promptscrape.py +++ b/promptscrape.py @@ -18,8 +18,6 @@ today = int(date.today().strftime("%d")) month = str(date.today().strftime("%B")) monthstring = ".*" + month + ".*" -prompts = [] - try: cent = "https://100words.dreamwidth.org/tag/!prompt?style=light&tag=%21prompt" centpage = requests.get(cent) @@ -35,7 +33,6 @@ try: centtheprompt = centprompttext.find("strong") print("100words (100 words): \033[1m" + centtheprompt.text.lower() + "\033[0m (" + centprompt + ")\n") thefile.write("- [[" + centprompt + "][100words]] (100 words): *" + centtheprompt.text.lower() + "*\n") - prompts.append({"source":"100words","type":"comm","notes":"100 words","prompt":centtheprompt.text.lower(),"url":centprompt}) except: pass @@ -91,7 +88,6 @@ try: adstrippable = adstrippable[1:] print("anythingdrabble (100, 200, 300, 400, or 500 words): \033[1m" + adstrippable.lower() + "\033[0m (" + adprompt + ")\n") thefile.write("- [[" + adprompt + "][anythingdrabble]] (100, 200, 300, 400, or 500 words): *" + adstrippable.lower() + "*\n") - prompts.append({"source":"anythingdrabble","type":"comm","notes":"100, 200, 300, 400, or 500 words","prompt":adstrippable.lower(),"url":adprompt}) except: pass @@ -99,7 +95,7 @@ try: dove = "https://dove-drabbles.dreamwidth.org/?style=light" dovepage = requests.get(dove) dovesoup = BeautifulSoup(dovepage.content, "html.parser") - doveprompts = dovesoup.find_all("h3", string=lambda text: "prompt" in text.lower()) + doveprompts = dovesoup.find_all("h3", string=lambda text: "prompt post" in text.lower()) dovesubsoup = BeautifulSoup(str(doveprompts[0]), "html.parser") doveurl = dovesubsoup.find("a") doveprompt = (doveurl["href"]) @@ -110,7 +106,6 @@ try: dovetheprompt = doveprompttext.find("i") print("dove-drabbles (any): \033[1m" + dovetheprompt.text.lower() + "\033[0m (" + doveprompt + ")\n") thefile.write("- [[" + doveprompt + "][dove-drabbles]] (any): *" + dovetheprompt.text.lower() + "*\n") - prompts.append({"source":"dove-drabbles","type":"comm","notes":"any length","prompt":dovetheprompt.text.lower(),"url":doveprompt}) except: pass @@ -128,16 +123,10 @@ try: zonepromptpage = s.get(zonepromptnew) zonepromptsoup = BeautifulSoup(zonepromptpage.content, "html.parser") zoneprompttext = zonepromptsoup.find(class_="entry-content") - zonetheprompt = zoneprompttext.find("div") + zonetheprompt = zoneprompttext.find("strong") print("drabble-zone (100 or 200 words): \033[1m" + zonetheprompt.text.lower() + "\033[0m (" + zoneprompt + ")\n") thefile.write("- [[" + zoneprompt + "][drabble-zone]] (100 or 200 words): *" + zonetheprompt.text.lower() + "*\n") - prompts.append({"source":"drabble-zone","type":"comm","notes":"100 or 200 words","prompt":zonetheprompt.text.lower(),"url":zoneprompt}) -except: - pass - -try: - with requests.Session() as s: - response = s.post(login_url , data) + emotion = "https://emotion100.dreamwidth.org/tag/*modpost?style=light&tag=%2Amodpost" emotionpage = s.get(emotion) emotionsoup = BeautifulSoup(emotionpage.content, "html.parser") @@ -152,7 +141,6 @@ try: emotiontheprompt = emotionprompttext.find_all("span")[-1] print("emotion100 (100 words or a multiple of 100): \033[1m" + emotiontheprompt.text.lower() + "\033[0m (" + emotionprompt + ")\n") thefile.write("- [[" + emotionprompt + "][emotion100]] (100 words or a multiple of 100): *" + emotiontheprompt.text.lower() + "*\n") - prompts.append({"source":"emotion100","type":"comm","notes":"100 words or a multiple of 100","prompt":emotiontheprompt.text.lower(),"url":emotionprompt}) except: pass @@ -196,7 +184,6 @@ try: ffaformat = "; ".join(ffacentnew) print("fail-fandomanon (any): \033[1m" + ffaformat.lower() + "\033[0m (" + ffaprompt + ")\n") thefile.write("- [[" + ffaprompt + "][fail-fandomanon]] (any): *" + ffaformat.lower() + "*\n") - prompts.append({"source":"fail-fandomanon","type":"comm","notes":"any length","prompt":ffacentnew,"url":ffaprompt}) except: pass @@ -219,7 +206,6 @@ try: fandomtheprompt = fandomprompttext.find("td") print("fandomweekly (any, competitive): \033[1m" + fandomtheprompt.text.lower() + "\033[0m (" + fandomprompt + ")\n") thefile.write("- [[" + fandomprompt + "][fandomweekly]] (any, competitive): *" + fandomtheprompt.text.lower() + "*\n") - prompts.append({"source":"fandomweekly","type":"comm","notes":"any length, competitive","prompt":fandomtheprompt.text.lower(),"url":fandomprompt}) except: pass @@ -238,28 +224,6 @@ try: flashtheprompt = flashprompttext.find("center") print("fan-flashworks (any, can’t post elsewhere until round is closed): \033[1m" + flashtheprompt.text.lower() + "\033[0m (" + flashprompt + ")\n") thefile.write("- [[" + flashprompt + "][fan-flashworks]] (any, can’t post elsewhere until round is closed): *" + flashtheprompt.text.lower() + "*\n") - prompts.append({"source":"fan-flashworks","type":"comm","notes":"any length, can’t post elsewhere until round is closed","prompt":flashtheprompt.text.lower(),"url":flashprompt}) -except: - pass - -try: - with requests.Session() as s: - response = s.post(login_url , data) - fsf = "https://femslashfete.dreamwidth.org/tag/admin?style=light&tag=admin" - fsfpage = s.get(fsf) - fsfsoup = BeautifulSoup(fsfpage.content, "html.parser") - fsfprompts = fsfsoup.find_all("h3", string=lambda text: "challenge" in text.lower()) - fsfsubsoup = BeautifulSoup(str(fsfprompts[0]), "html.parser") - fsfurl = fsfsubsoup.find("a") - fsfprompt = (fsfurl["href"]) - fsfpromptnew = (fsfurl["href"] + "?style=light") - fsfpromptpage = s.get(fsfpromptnew) - fsfpromptsoup = BeautifulSoup(fsfpromptpage.content, "html.parser") - fsfprompttext = fsfpromptsoup.find(class_="entry-content") - fsftheprompt = fsfprompttext.find("b") - print("femslashfete (at least 100 words, must be femslash): \033[1m" + fsftheprompt.text.lower() + "\033[0m (" + fsfprompt + ")\n") - thefile.write("- [[" + fsfprompt + "][femslashfete]] (at least 100 words, must be femslash): *" + fsftheprompt.text.lower() + "*\n") - prompts.append({"source":"femslashfete","type":"comm","notes":"at least 100 words, must be femslash","prompt":fsftheprompt.text.lower(),"url":fsfprompt}) except: pass @@ -300,7 +264,6 @@ try: fffclittleprompttext = fffclittlepromptsoup.find("h3") print("fffc little special (at least 100 words): \033[1m" + fffclittleprompttext.text.lower() + "\033[0m (" + fffclittleprompt + ")\n") thefile.write("- [[" + fffclittleprompt + "][fffc little special]] (at least 100 words): *" + fffclittleprompttext.text.lower() + "*\n") - prompts.append({"source":"fffc","type":"comm","notes":"at least 100 words","prompt":fffclittleprompttext.text.lower(),"url":fffclittleprompt,"challenge":"little special"}) fffcmadnessprompts = fffcsoup.find_all("h3", string=lambda text: "froday madness" in text.lower()) fffcmadnesssubsoup = BeautifulSoup(str(fffcmadnessprompts[0]), "html.parser") fffcmadnessurl = fffcmadnesssubsoup.find("a") @@ -312,7 +275,6 @@ try: fffcmadnesstheprompt = fffcmadnessprompttext.find("b") print("fffc madness (at least 2000 words): \033[1m" + fffcmadnesstheprompt.text.lower() + "\033[0m (" + fffcmadnessprompt + ")\n") thefile.write("- [[" + fffcmadnessprompt + "][fffc madness]] (at least 2000 words): *" + fffcmadnesstheprompt.text.lower() + "*\n") - prompts.append({"source":"fffc","type":"comm","notes":"at least 2000 words","prompt":fffcmadnesstheprompt.text.lower(),"url":fffcmadnessprompt,"challenge":"froday madness"}) fffcmonthlyprompts = fffcsoup.find_all("h3", string=re.compile(monthstring)) fffcmonthlysubsoup = BeautifulSoup(str(fffcmonthlyprompts[0]), "html.parser") fffcmonthlyurl = fffcmonthlysubsoup.find("a") @@ -323,7 +285,6 @@ try: fffcmonthlyprompttext = fffcmonthlypromptsoup.find("h3") print("fffc monthly special (usually at least 500 words): \033[1m" + fffcmonthlyprompttext.text.lower() + "\033[0m (" + fffcmonthlyprompt + ")\n") thefile.write("- [[" + fffcmonthlyprompt + "][fffc monthly special]] (usually at least 500 words): *" + fffcmonthlyprompttext.text.lower() + "*\n") - prompts.append({"source":"fffc","type":"comm","notes":"usually at least 500 words","prompt":fffcmonthlyprompttext.text.lower(),"url":fffcmonthlyprompt,"challenge":"monthly challenge"}) fffcregularprompts = fffcsoup.find_all("h3", string=lambda text: "regular challenge" in text.lower()) fffcregularsubsoup = BeautifulSoup(str(fffcregularprompts[0]), "html.parser") fffcregularurl = fffcregularsubsoup.find("a") @@ -335,7 +296,6 @@ try: fffcregulartheprompt = fffcregularprompttext.find("b") print("fffc regular challenge (at least 100 words): \033[1m" + fffcregulartheprompt.text.lower() + "\033[0m (" + fffcregularprompt + ")\n") thefile.write("- [[" + fffcregularprompt + "][fffc regular challenge]] (at least 100 words): *" + fffcregulartheprompt.text.lower() + "*\n") - prompts.append({"source":"fffc","type":"comm","notes":"at least 100 words","prompt":fffcregulartheprompt.text.lower(),"url":fffcregularprompt,"challenge":"regular challenge"}) except: pass @@ -354,7 +314,6 @@ try: ficlettheprompt = ficletprompttext.find("a") print("ficlet-zone (any): \033[1m" + ficlettheprompt.text.lower() + "\033[0m (" + ficletprompt + ")\n") thefile.write("- [[" + ficletprompt + "][ficlet-zone]] (any): *" + ficlettheprompt.text.lower() + "*\n") - prompts.append({"source":"ficlet-zone","type":"comm","notes":"any length","prompt":ficlettheprompt.text.lower(),"url":ficletprompt}) except: pass @@ -381,7 +340,6 @@ try: hourlypromptthishour = str(hourlypromptmedian[0])[5:-5] print("hourlyprompts (any): \033[1m" + hourlypromptthishour.lower() + "\033[0m (" + hourlyprompt + ")\n") thefile.write("- [[" + hourlyprompt + "][hourlyprompts]] (any): *" + hourlypromptthishour.lower() + "*\n") - prompts.append({"source":"hourlyprompts","type":"comm","notes":"any length","prompt":hourlypromptthishour.lower(),"url":hourlyprompt}) except: pass @@ -423,7 +381,6 @@ if themonth != 4 and themonth != 8 and themonth != 12: ssbingofinal = "; ".join(ssbingoclean).lower() print("sweet and short bingo (up to 500 words, separate or combined): \033[1m" + ssbingofinal + "\033[0m (" + ssbingoprompt + ")\n") thefile.write("- [[" + ssbingoprompt + "][sweet and short bingo]] (up to 500 words, separate or combined): *" + ssbingofinal + "*\n") - prompts.append({"source":"sweetandshort","type":"comm","notes":"up to 500 words, separate or combined","prompt":ssbingoclean,"url":ssbingoprompt,"challenge":"bingo"}) except: pass @@ -441,18 +398,18 @@ if themonth != 4 and themonth != 8 and themonth != 12: ssquickypromptnew = (ssquickyurl["href"]) ssquickypromptpage = requests.get(ssquickypromptnew) ssquickypromptsoup = BeautifulSoup(ssquickypromptpage.content, "html.parser") + promptcatch = ".*New Prompts Here" + # ssquickytheprompt = ssquickypromptsoup.find_all("h4",string = re.compile(promptcatch)) ssquickytheprompt = ssquickypromptsoup.find_all(class_="comment") ssquickycomments = [] for comment in ssquickytheprompt: - if re.search("new prompts here",str(comment),re.IGNORECASE): + if re.search("New Prompts Here",str(comment)): commenttext = re.findall(r"
Multifandom prompt communities on DW
\n
" + prompt["source"].replace("-","_") + "")
- try:
- if prompt["challenge"]:
- htmlfile.write(" " + prompt["challenge"])
- except:
- pass
- if type(prompt["prompt"]) == list:
- htmlfile.write(": prompt post (" + prompt["notes"] + ")
\n\n") - for theprompt in prompt["prompt"]: - htmlfile.write("- " + theprompt + "
\n")
- htmlfile.write("
") - else: - htmlfile.write(": " + prompt["prompt"] + " (" + prompt["notes"] + ")") - htmlfile.write("Generated " + datetime.now().strftime("%H:%M (UK time), %-d %B %Y") + "