Python
interencheres
import re, os, uuid, datetime, sys, signal
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.remote.remote_connection import LOGGER as seleniumLogger
import logging
from subprocess import CREATE_NO_WINDOW
from time import sleep
import urllib.request
from pathlib import Path
## variables
service = Service('Q:/WORK/TEMP/python/chromedriver_win32/chromedriver.exe')
service.creationflags = CREATE_NO_WINDOW
subfolder = 'encheres'
looptime = 5
## defs
def log(text,sameline=False):
if sameline:
print(text,end = '\r')
else:
print(text)
def sigint_handler(signal, frame):
log('KeyboardInterrupt is caught')
driver.quit()
sys.exit(0)
signal.signal(signal.SIGINT, sigint_handler)
def touch(fname, times=None):
Path(fname).mkdir(parents=True, exist_ok=True)
def textappend(file,text):
with open(file, 'a',encoding='utf-8-sig') as record:
record.write(text+'\n')
def download(url,savedir):
touch(savedir)
uid = uuid.uuid4().hex
urllib.request.urlretrieve(url, savedir+'/'+uid+'.jpg')
return uid
def out(driver):
driver.quit()
exit()
def createHtml(textfile):
htmlfile = textfile+'.html'
if '.txt' in htmlfile:
try:
os.remove(htmlfile)
except OSError:
log('cannot delete'+ htmlfile)
pass
switch = 0
colors = ['#F4F4F4','#FBFBFB']
html = '<html><body><table>'
with open(textfile, 'r',encoding='utf-8-sig') as records:
page = records.readlines()
for item in page:
switch = not switch
L = item.split(' | ')
image = '<img style="max-height:200px;max-width:200px" src="images/'+L[0]+'.jpg">'
infos = L[3]+'<br/>\n\t<br/>\n\t'+L[2]+': '+L[4]
html += '<tr style="background-color:{color}">\n\t<td align="right">{image}</td>\n\t<td valign="top" style="padding:10px"><strong>{price}</strong><br/>\n\t{infos}\t</td>\n</tr>\n'.format(image=image,price=L[1],infos=infos,color=colors[switch])
html += '</table></body></html>'
try:
textappend(htmlfile,html)
log('writing html file')
except:
log('error writing html file')
def writeOut(driver,previousprice,previousiteminfos,filepath,looptime):
outstring = uid+' | '+previousprice+' | '+' | '.join(previousiteminfos);
log(outstring)
textappend(filepath,outstring)
createHtml(filepath)
log('looping every '+str(looptime)+'s')
url = ''
try:
url = sys.argv[1]
except:
log('url needed as argument')
exit()
savepath = os.path.dirname(os.path.realpath(__file__))+'/'+subfolder
fileNameDate = datetime.datetime.now().strftime("%y%m%d_%H-%M")
filepath = savepath+'/'+fileNameDate+'_'+url.split('/')[3]+'_'+url.split('/')[4]+'.txt'
log('loading headless chrome...')
seleniumLogger.setLevel(logging.WARNING)
options = webdriver.ChromeOptions()
#options.add_experimental_option('prefs', prefs)
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument('--window-size=1280x1696')
options.add_argument('--user-data-dir=/tmp/user-data')
options.add_argument('--hide-scrollbars')
options.add_argument('--service-log-path='+os.devnull)
options.add_argument('--enable-logging')
options.add_argument('--log-level=3')
options.add_argument('--silent');
options.add_argument('--v=99')
options.add_argument('--single-process')
options.add_argument('--data-path=/tmp/data-path')
options.add_argument('--ignore-certificate-errors')
options.add_argument('--homedir=/tmp')
options.add_argument('--disk-cache-dir=/tmp/cache-dir')
options.add_argument('user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36')
driver = webdriver.Chrome(service=service,options=options)
isWaitingToStart = True
isOn = True
url = driver.get(url)
log('loading ' + driver.current_url) #sanity check
log('writing to '+filepath)
# figure our dom elements
# is it started or over, if not wait
while isWaitingToStart:
try:
items = driver.find_elements(By.CLASS_NAME, 'wrapper-join-sale-button')
for item in items:
if item.text == 'Cette vente est terminée':
log('over')
out(driver)
elif 'La vente commence dans' in item.text:
log('waiting to start...')
sleep(100)
else:
isWaitingToStart = False
log('starting')
break
except NoSuchElementException:
print('No wrapper-join-sale-button')
# assume it's on
try:
priceitem = driver.find_element(By.CLASS_NAME, 'text-h5')
items = driver.find_elements(By.CLASS_NAME, "text-h5")
for item in items:
if '€' in item.text:
priceitem = item
break
except NoSuchElementException:
log('No text-h5 found')
out(driver)
try:
itemdiv = driver.find_element(By.CLASS_NAME, 'current-item')
except NoSuchElementException:
log('No current-item')
out(driver)
# turn off video if it's on
try:
driver.execute_script('document.getElementById("streaming-subscriber").pause();')
log('video paused')
except:
log('No video stream found')
previousprice = 'none'
previousitem = 'none'
previousiteminfos = []
previousimage = 'none'
uid = 'none'
# loop until we break
while isOn:
# price and/or stop if finished
try:
findprice = driver.find_element(By.CLASS_NAME, 'text-h5')
if not findprice:
isOn = False
log('finished, stopping')
out(driver)
fetchprice = priceitem.text
fetchprice = "".join(fetchprice.split()) # remove weird space characters
except:
log('price error, most likely finished')
writeOut(driver,previousprice,previousiteminfos,filepath,looptime)
#createHtml(filepath)
out(driver)
# current item description and image
try:
currentiteminfos = itemdiv.text.splitlines()
currentitem = currentiteminfos[0]
except:
log('item error')
out(driver)
#fetch image
try:
rgx = r'background-image: url\("(.*)"'
images = itemdiv.find_elements(By.CLASS_NAME, 'v-image__image')
for image in images:
style = image.get_attribute('style')
result = re.search(rgx, style)
if result:
currentimage = result.groups(0)[0]
if currentimage != previousimage and previousimage != 'none':
uid = download(previousimage,savepath+'/images')
log('new image downloaded')
previousimage = currentimage
except:
log('image error')
out(driver)
# if things have changed, it means the previous item bid is over
if (currentitem != previousitem ) and previousprice != 'none':
writeOut(driver,previousprice,previousiteminfos,filepath,looptime)
#outstring = uid+' | '+previousprice+' | '+' | '.join(previousiteminfos);
#log(outstring)
#textappend(filepath,outstring)
#createHtml(filepath)
#log('looping every '+str(looptime)+'s')
try:
driver.execute_script('document.getElementById("streaming-subscriber").pause();')
log('video paused')
except:
log('no video stream found')
if previousprice != fetchprice:
log(previousprice+' ',True)
if fetchprice != '-- €':
previousprice = fetchprice
previousitem = currentitem
previousiteminfos = currentiteminfos
previousimage = currentimage
sleep(looptime)
driver.quit()
log('script finished')
import os
def touch(fname, times=None):
Path(fname).mkdir(parents=True, exist_ok=True)
def textappend(file,text):
with open(file, 'a',encoding='utf-8-sig') as record:
record.write(text+'\n')
subfolder = 'encheres'
savepath = os.path.dirname(os.path.realpath(__file__))+'/'+subfolder
txtfile = savepath+'/220217_12-11_materiels-professionnels_vehicules-et-materiel-304020.txt'
def createHtml(textfile):
htmlfile = textfile+'.html'
if '.txt' in htmlfile:
try:
os.remove(htmlfile)
except OSError:
#log('cannot delete'+ htmlfile)
pass
switch = 0
colors = ['#F4F4F4','#FBFBFB']
html = '<html><body><table>'
with open(textfile, 'r',encoding='utf-8-sig') as records:
page = records.readlines()
for item in page:
switch = not switch
L = item.split(' | ')
image = '<img style="max-height:200px;max-width:200px" src="images/'+L[0]+'.jpg">'
infos = L[3]+'<br/>\n\t<br/>\n\t'+L[2]+': '+L[4]
html += '<tr style="background-color:{color}">\n\t<td align="right">{image}</td>\n\t<td valign="top" style="padding:10px"><strong>{price}</strong><br/>\n\t{infos}\t</td>\n</tr>\n'.format(image=image,price=L[1],infos=infos,color=colors[switch])
html += '</table></body></html>'
try:
textappend(htmlfile,html)
log('writing html file')
except:
log('error writing html file')
createHtml(txtfile)
Radios scraping
todo: web<>PI interface
import json
import urllib.request
import urllib.parse
import os
import datetime
from pathlib import Path
#todo: make sure tail works without opening ressource entirely
path = os.path.dirname(os.path.realpath(__file__))
#path = os.getcwd()
sep = " | "
date = datetime.datetime.now().strftime("%y%m%d")
def touch(fname, times=None):
with open(fname, 'a'):
os.utime(fname,times)
def tail(f, lines=1, _buffer=4098):
'''Tail a file and get X lines from the end'''
lines_found = []
block_counter = -1
file = open(f,'r')
while len(lines_found) < lines:
try:
file.seek(block_counter * _buffer, os.SEEK_END)
except IOError: # either file is too small, or too many lines requested
file.seek(0)
lines_found = file.readlines()
break
lines_found = file.readlines()
block_counter -= 1
return lines_found[-lines:]
file.close()
def getlastlines(file,n):
''' https://stackoverflow.com/questions/46258499/how-to-read-the-last-line-of-a-file-in-python '''
with open(file, 'r') as f:
lines = f.readlines()
return lines[-n:]
#####################
# #
# RADIO MEUH #
# #
#####################
url = "https://www.radiomeuh.com/player/rtdata/tracks.json"
#localfile = path+"/home/bernie/tools/loop/data/playlist.txt"
localfile = path+"/data/playlist_meuh.txt"
touch(localfile)
lastlines = getlastlines(localfile,10)
content = urllib.request.urlopen(url)
data = json.loads(content.read())
for el in reversed(data):
record = ("{date}{sep}{time}{sep}{artist}{sep}{titre}".format(sep=sep,date=date,time=el['time'],artist=el['artist'],titre=el['titre']))
artisttitre = str(el['artist']+sep+el['titre'])
if len(lastlines) < 2:
lastlines = ['0','0']
#if (record != lastlines[0].strip()) and (record != lastlines[1].strip()):
if not any(artisttitre in s for s in lastlines):
with open(localfile, 'a',encoding='utf-8-sig') as recordlist:
recordlist.write(record+'\n')
#####################
# #
# ELLEBORE #
# #
#####################
url = 'https://www.radio-ellebore.com/wp-admin/admin-ajax.php'
localfile = path+"/data/playlist_ellebore.txt"
touch(localfile)
lastline = getlastlines(localfile,1)
if len(lastline)==0:
lastline = ['0 | 0 | 0 | 0']
lastlog = lastline[0].strip().split(' | ')
data = {'action':'get_stream_title','stream':'https://ellebore.ice.infomaniak.ch/ellebore-high.aac'}
data = urllib.parse.urlencode(data).encode()
hdr = { 'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8' }
req = urllib.request.Request(url, data=data, headers=hdr,method='POST')
response = urllib.request.urlopen(req)
result = response.read().decode('utf-8')
result = result.split(' - ')
time = datetime.datetime.now().strftime("%H:%M:%S")
record = ("{date}{sep}{time}{sep}{artist}{sep}{titre}".format(sep=sep,date=date,time=time,artist=result[0].strip(),titre=result[1].strip()))
if lastlog[-1].strip() == '|':
lastlog = ['a','b','c','d']
if str(result[0]+result[1]) != str(lastlog[2]+lastlog[3]) or lastline[0] == '0 | 0 | 0 | 0':
with open(localfile, 'a',encoding='utf-8-sig') as recordlist:
recordlist.write(record+'\n')
#le djam https://www.djamradio.com/actions/infos.php
#https://www.djamradio.com/actions/retrieve.php
Create a chart with frame size and render time
py2
Requires matplotlib to make charts, runs on right-clicking a file part of a sequence usign the windows sendto>menu (that you can edit by doing Win-R > shell:sendto)
python -m pip install -U pip python -m pip install -U matplotlib
Creates a chart with render time and frame size using file creation time (so as it was pointed out a little useless for multi-machine renders). Useful if you want to get a glimpse of simulation frame times
import os, sys, re, time
import matplotlib.pyplot as plt
import numpy as np
def sequence(file):
'''given a file path, return a dictionnary with [directory, filename (without number and extension), start frame, end frame, padding, extension]'''
# given C:/path/the_sequence_0033.jpg
# like so ['c:/path/','the_sequence_',1,100,4,'.jpg']
if os.path.isfile(file):
reg = r'^(.+?)([0-9]+)\.([.a-zA-Z]{1,7})$'
match = re.match(reg, file,re.IGNORECASE)
if match:
#return target
newReg = r'('+os.path.basename(match.groups()[0])+')(\d*)\.('+match.groups()[2]+')'
#bit convoluted but it will help me pick the first image of sequence that matches selection
filelist = []
target = os.path.dirname(file)
for f in os.listdir(target):
match = re.match(newReg, f,re.IGNORECASE)
if match:
filelist.append(match.groups())
return [ target , filelist[0][0] , int(filelist[0][1]) , int(filelist[-1][1]) , len(filelist[0][1]) , filelist[0][2] ]
sequence = sequence(sys.argv[1])
start = sequence[2]
end = sequence[3]
sequenceInfos = []
for n in range(start,end+1):
file = '{folder}/{file}{number}.{extension}'.format( folder = sequence[0], file = sequence[1], number = str(n).zfill( sequence[4] ), extension = sequence[5] )
file = os.path.abspath(file)
creationTime = os.path.getctime(file)
size = os.path.getsize(file)
sequenceInfos.append([file,creationTime,size])
frames = list(range(start, end + 1))
times = [ x[1] for x in sequenceInfos ] #creation times
times_temp = times.copy()
for i in range(1,len(times)):
times_temp[i] = times[i] - times[i-1]
times_temp[0] = times_temp[1]
times = times_temp.copy()
sizes = [ x[2] / 1024.0 / 1024.0 for x in sequenceInfos ]
fig, ax1 = plt.subplots()
color = 'tab:red'
ax1.set_xlabel('frames')
ax1.set_ylabel('Frame Time (s)', color=color)
ax1.plot(frames, times, color=color)
ax1.tick_params(axis='y', labelcolor=color)
ax1.set_ylim(bottom=0)
ax2 = ax1.twinx()
color = 'tab:blue'
ax2.set_ylabel('Size (mb)', color=color) # we already handled the x-label with ax1
ax2.plot(frames, sizes, color=color)
ax2.tick_params(axis='y', labelcolor=color)
ax2.set_ylim(bottom=0)
fig.tight_layout()
plt.grid()
plt.show()
Houdini remote killswitch
import urllib.request
import os
import time
from random import random
while True:
url = "https://berniebernie.fr/tools/stamp/up.txt"
url += "?"+str(random())
contents = urllib.request.urlopen(url).read()
contents = contents.decode()
if contents=='kill':
os.system('taskkill /F /IM "houdini*"')
print('killed')
break
else:
print(url)
print(contents)
time.sleep(30)
Create a folder with today's date
import os
import time
from datetime import date
today = date.today()
folder = today.strftime("%Y_%m_%d")
folder = os.getcwd()+'/'+folder
try:
os.mkdir(folder)
print("creating "+folder)
except:
print(folder+ " exists or failed")
time.sleep(.8)
Find the name of the cloest matching color
def colorName(r,g,b):
'''returns the name of the closest color from the list using distance from r,g,b so might no be super precise :) '''
colors = {
'AliceBlue':[240,248,255],
'AntiqueWhite':[250,235,215],
'Aqua':[0,255,255],
'Aquamarine':[127,255,212],
'Azure':[240,255,255],
'Beige':[245,245,220],
'Bisque':[255,228,196],
'Black':[0,0,0],
'BlanchedAlmond':[255,235,205],
'Blue':[0,0,255],
'BlueViolet':[138,43,226],
'Brown':[165,42,42],
'BurlyWood':[222,184,135],
'CadetBlue':[95,158,160],
'Chartreuse':[127,255,0],
'Chocolate':[210,105,30],
'Coral':[255,127,80],
'CornflowerBlue':[100,149,237],
'Cornsilk':[255,248,220],
'Crimson':[220,20,60],
'Cyan':[0,255,255],
'DarkBlue':[0,0,139],
'DarkCyan':[0,139,139],
'DarkGoldenRod':[184,134,11],
'DarkGray':[169,169,169],
'DarkGrey':[169,169,169],
'DarkGreen':[0,100,0],
'DarkKhaki':[189,183,107],
'DarkMagenta':[139,0,139],
'DarkOliveGreen':[85,107,47],
'DarkOrange':[255,140,0],
'DarkOrchid':[153,50,204],
'DarkRed':[139,0,0],
'DarkSalmon':[233,150,122],
'DarkSeaGreen':[143,188,143],
'DarkSlateBlue':[72,61,139],
'DarkSlateGray':[47,79,79],
'DarkSlateGrey':[47,79,79],
'DarkTurquoise':[0,206,209],
'DarkViolet':[148,0,211],
'DeepPink':[255,20,147],
'DeepSkyBlue':[0,191,255],
'DimGray':[105,105,105],
'DimGrey':[105,105,105],
'DodgerBlue':[30,144,255],
'FireBrick':[178,34,34],
'FloralWhite':[255,250,240],
'ForestGreen':[34,139,34],
'Fuchsia':[255,0,255],
'Gainsboro':[220,220,220],
'GhostWhite':[248,248,255],
'Gold':[255,215,0],
'GoldenRod':[218,165,32],
'Gray':[128,128,128],
'Grey':[128,128,128],
'Green':[0,128,0],
'GreenYellow':[173,255,47],
'HoneyDew':[240,255,240],
'HotPink':[255,105,180],
'IndianRed ':[205,92,92],
'Indigo ':[75,0,130],
'Ivory':[255,255,240],
'Khaki':[240,230,140],
'Lavender':[230,230,250],
'LavenderBlush':[255,240,245],
'LawnGreen':[124,252,0],
'LemonChiffon':[255,250,205],
'LightBlue':[173,216,230],
'LightCoral':[240,128,128],
'LightCyan':[224,255,255],
'LightGoldenRodYellow':[250,250,210],
'LightGray':[211,211,211],
'LightGrey':[211,211,211],
'LightGreen':[144,238,144],
'LightPink':[255,182,193],
'LightSalmon':[255,160,122],
'LightSeaGreen':[32,178,170],
'LightSkyBlue':[135,206,250],
'LightSlateGray':[119,136,153],
'LightSlateGrey':[119,136,153],
'LightSteelBlue':[176,196,222],
'LightYellow':[255,255,224],
'Lime':[0,255,0],
'LimeGreen':[50,205,50],
'Linen':[250,240,230],
'Magenta':[255,0,255],
'Maroon':[128,0,0],
'MediumAquaMarine':[102,205,170],
'MediumBlue':[0,0,205],
'MediumOrchid':[186,85,211],
'MediumPurple':[147,112,219],
'MediumSeaGreen':[60,179,113],
'MediumSlateBlue':[123,104,238],
'MediumSpringGreen':[0,250,154],
'MediumTurquoise':[72,209,204],
'MediumVioletRed':[199,21,133],
'MidnightBlue':[25,25,112],
'MintCream':[245,255,250],
'MistyRose':[255,228,225],
'Moccasin':[255,228,181],
'NavajoWhite':[255,222,173],
'Navy':[0,0,128],
'OldLace':[253,245,230],
'Olive':[128,128,0],
'OliveDrab':[107,142,35],
'Orange':[255,165,0],
'OrangeRed':[255,69,0],
'Orchid':[218,112,214],
'PaleGoldenRod':[238,232,170],
'PaleGreen':[152,251,152],
'PaleTurquoise':[175,238,238],
'PaleVioletRed':[219,112,147],
'PapayaWhip':[255,239,213],
'PeachPuff':[255,218,185],
'Peru':[205,133,63],
'Pink':[255,192,203],
'Plum':[221,160,221],
'PowderBlue':[176,224,230],
'Purple':[128,0,128],
'RebeccaPurple':[102,51,153],
'Red':[255,0,0],
'RosyBrown':[188,143,143],
'RoyalBlue':[65,105,225],
'SaddleBrown':[139,69,19],
'Salmon':[250,128,114],
'SandyBrown':[244,164,96],
'SeaGreen':[46,139,87],
'SeaShell':[255,245,238],
'Sienna':[160,82,45],
'Silver':[192,192,192],
'SkyBlue':[135,206,235],
'SlateBlue':[106,90,205],
'SlateGray':[112,128,144],
'SlateGrey':[112,128,144],
'Snow':[255,250,250],
'SpringGreen':[0,255,127],
'SteelBlue':[70,130,180],
'Tan':[210,180,140],
'Teal':[0,128,128],
'Thistle':[216,191,216],
'Tomato':[255,99,71],
'Turquoise':[64,224,208],
'Violet':[238,130,238],
'Wheat':[245,222,179],
'White':[255,255,255],
'WhiteSmoke':[245,245,245],
'Yellow':[255,255,0],
'YellowGreen':[154,205,50]
}
testColor = [25,245,1]
closestColor = 'None'
diff = 99999999
for color in colors:
d = pow(colors[color][0] - r,2)
d = d + pow(colors[color][1] - g,2)
d = d + pow(colors[color][2] - b,2)
if min(diff,d) == d:
diff = d
closestColor = color
return closestColor
print(colorName(254,50,23))
List all objs in directory and subdirectories to text file
import os
dir_path = os.path.dirname(os.path.realpath(__file__))
listfilename = 'objs_list.txt'
objlist = []
for root, dirs, files in os.walk(dir_path):
for file in files:
if file.lower().endswith('.obj'):
objlist.append(os.path.join(root, file))
print('Writing '+str(len(objlist))+'to '+dir_path+'/'+listfilename)
with open(dir_path+'/'+listfilename, 'w') as file_handler:
for item in objlist:
file_handler.write("{}\n".format(item))
elections 2017 scraping
# # -*- coding: utf-8 -*-
from urllib import urlopen
import re
import urlparse
import os
baseUrl = 'http://elections.interieur.gouv.fr/presidentielle-2017/'
townUrlsFile = os.path.dirname(os.path.realpath(__file__))+'/town_urls.txt'
#print townUrlsFile
def getTownUrlsList():
with open(townUrlsFile, 'w') as fid:
count = 0
townlist = []
#get the page
page = urlopen(baseUrl+'index.html')
page_content_HTML = page.read()
#grab the list
start = 'selected>Choisir un département</option>'
end = '</select><br><p class="clic-carte">'
departement_HTML = (page_content_HTML.split(start))[1].split(end)[0]
#iterate through departments (options)
options = re.findall(r'<option value="(.*)">(.*)</option>',departement_HTML,re.M)
for option in options:
#get the page
page = urlopen(baseUrl+option[0])
page_content_HTML = page.read()
#grab the list of town letters
start = 'initiale</i><br>'
end = '\xa0\n\t\t\t<hr>\n</div></div>\n<div class="row-fluid pub-index-communes">'
town_letters_HTML = (page_content_HTML.split(start))[1].split(end)[0]
#iterate through town letters (A = all towns with A in this departement etc...)
town_letters = re.findall(r'<a href="../../(.*)">(.*)</a>',town_letters_HTML,re.M)
for town_letter in town_letters:
page = urlopen(baseUrl+town_letter[0])
page_content_HTML = page.read()
#grab the list of towns
start = 'tableau-communes"><tbody>'
end = '</tbody></table>\n<br>\n</div></div>\n</div>\n<div class="row-fluid pub-bas">\n<div class="span5">'
towns_HTML = (page_content_HTML.split(start))[1].split(end)[0]
#print towns_HTML
towns = re.findall(r'<tr><td><a href="../../(.*)">(.*)</a>',towns_HTML,re.M)
#iterate through towns
for town in towns:
currentTown = option[1]+'|'+town[1]+'|'+baseUrl+town[0]
#townlist.append()
fid.write(currentTown+'\n')
count = count + 1
print count
#return townlist
#fid.write('\n'.join(getTownUrlsList()))
#fid.close()
getTownUrlsList()
# # -*- coding: utf-8 -*-
from urllib import urlopen
import re
import urlparse
import os
page = urlopen('http://elections.interieur.gouv.fr/presidentielle-2017/011/075/index.html')
page_content_HTML = page.read()
#grab the list of town letters
if 'arrondissement</i></p>' in page_content_HTML:
start = 'arrondissement</i></p>'
end = '<div class="row-fluid pub-resultats-entete'
else:
start = 'initiale</i><br>'
end = '\xa0\n\t\t\t<hr>\n</div></div>\n<div class="row-fluid pub-index-communes">'
town_letters_HTML = (page_content_HTML.split(start))[1].split(end)[0]
for arrondissement in town_letters_HTML.split('</a> <a href'):
print arrondissement.split('../../')[1].split('">')
#town_letters = re.findall(r'<a href="../../(.*)">(.*)</a>',town_letters_HTML,re.M)
#for town_letter in town_letters:
#print town_letter
colorlovers color scraping
# # -*- coding: utf-8 -*-
from urllib.request import urlopen
import os
import codecs
import math
import time
def grabCL(n):
url = "http://www.colourlovers.com/ajax/browse-palettes/_page_"+str(n)+"?section=most-loved&period=all-time&view=meta&channelID=0"
page = urlopen(url)
page_content = page.read()
with open('Z:/BERNIE/vvvv/palettes/cl/output'+str(n)+'.txt', 'w') as fid:
fid.write(str(page_content))
fid.close()
def parseCL(n):
''' disgusting code to parse webpage because i can't figure out beautifulsoup '''
output = ""
titles = []
with open('Z:/BERNIE/vvvv/palettes/cl/output'+str(n)+'.txt', 'r') as fid:
for line in fid:
tokens = line.split("</a></h3><div class=\"left username\"")
p = len(tokens)
for i in range(p):
tokensTitle = tokens[i].split("\">")
titles.append(tokensTitle[-1])
#get colors
lines = line.split("<span class=\"c\" style=\"width: ")
j = 1
while j<len(lines):
#print(titles[(int((j-1)/10))])
output += "\n"+str(int((j-1)/10)+(n-1)*15)+" "+titles[(int((j-1)/10))].replace("\\", "")+"\n"
for k in range(5):
curline = lines[j+k]
widthTokens = curline.split("px; height: 50px;")
width = widthTokens[0]
colorTokens = curline.split(";\"><span class=\"s\" style=\"margin-top: 45px;\">")
color = colorTokens[0][-7:]
colorString = color+" "+str(float(width)/560)[:6]
#print(colorString)
output += colorString+"\n"
#output += "\n"
j = j + 10
return output
def scrapeCL(startPage, endPage, waitInSeconds):
for i in range(startPage, endPage+1):
grabCL(i)
out = parseCL(i)
#print(str(i))
with open("Z:/BERNIE/vvvv/palettes/colors_cl.txt", "a") as myfile:
myfile.write(out)
print("Page "+str(i)+" grabbed... "+str(i*15)+ " records")
time.sleep(waitInSeconds)
scrapeCL(1,270,1)
Raytracer
from PIL import Image
from math import sqrt
imwidth = 640
imheight = 480
im = Image.new("RGB",(imwidth,imheight),"black")
#im = Image.open("lolmonkey.jpg")
#im.show()
#im.save("hellwrld.png","PNG")
#NUMPY
mysphere = [[0,0,0],200]
#myray = [[0,0,10],[0,0,-1]]
def intersectRaySphere(ray,sphere):
A = ray[1][0]*ray[1][0] + ray[1][1]*ray[1][1] + ray[1][2]*ray[1][2]
B = 2.0 * (ray[1][0]*(ray[0][0]-sphere[0][0]) + ray[1][1]*(ray[0][1]-sphere[0][1]) + ray[1][2]*(ray[0][2]-sphere[0][2]))
C = (ray[0][0]-sphere[0][0])*(ray[0][0]-sphere[0][0]) + (ray[0][1]-sphere[0][1])*(ray[0][1]-sphere[0][1]) + (ray[0][2]-sphere[0][2])*(ray[0][2]-sphere[0][2]) - sphere[1]
delta = B*B - 4.0*A*C
results = []
if(delta==0):
results.append(-B/(2.0*A))
if(delta>0):
results.append((-B+(sqrt(delta)))/(2.0*A))
results.append((-B-(sqrt(delta)))/(2.0*A))
points = []
for t in results:
points.append([ray[0][0] + t*ray[1][0], ray[0][1] + t*ray[1][1], ray[0][2] + t*ray[1][2]])
if(len(points)==2):
if(points[0]>points[1]):
points = points[1]
else:
points = points[0]
return points
def mag(vec):
return sqrt(vec[0]*vec[0] + vec[1]*vec[1] + vec[2]*vec[2]);
def cross(vec1,vec2):
return [vec1[1]*vec2[2]-vec1[2]*vec2[1], vec1[2]*vec2[0]-vec1[0]*vec2[2], vec1[0]*vec2[1]-vec1[1]*vec2[0]]
def normalize(vec):
return [vec[0]/mag(vec),vec[1]/mag(vec),vec[2]/mag(vec)]
def dot(vec1,vec2):
return vec1[0]*vec2[0] + vec1[1]*vec2[1] + vec1[2]*vec2[2]
def pixToPoint(i,j,width,height,xPixSize,yPixSize,center,u,v):
cu = (float(2*i+1)/(2*xPixSize)-.5)*width
cv = (float(2*j+1)/(2*yPixSize)-.5)*height
x = center[0]+cu*u[0]+cv*v[0]
y = center[1]+cu*u[1]+cv*v[1]
z = center[2]+cu*u[2]+cv*v[2]
#print [x,y,z]
#print [i,j]
#print [cu,cv]
return [x,y,z]
lookat = [0,0,0]
eye = [100,100,100]
f = 10
upvector = [0,1,0]
viewplaneW = imwidth/2
viewplaneH = imheight/2
EA = [lookat[0]-eye[0],lookat[1]-eye[1],lookat[2]-eye[2]]
lenEA = mag(EA)
normEA = [EA[0]/lenEA,EA[1]/lenEA,EA[2]/lenEA]
center = [EA[0]+normEA[0]*f, EA[1]+normEA[1]*f, EA[2]+normEA[2]*f]
w = normEA
u = normalize(cross(upvector,w))
v = normalize(cross(u,w))
#print(cross([1,0,0],[0,1,0]))
light = [0,0,100]
#print intersectRaySphere(myray,mysphere)
for x in range(imwidth):
for y in range(imheight):
#myray = [[x,y,-10],[0,0,1]]
point = pixToPoint(x,y,imwidth,imheight,viewplaneW,viewplaneH,center,u,v)
ray = [point,[point[0]-eye[0] , point[1]-eye[1] , point[2]-eye[2]]]
if(len(intersectRaySphere(ray,mysphere))):
n = normalize([point[0]-mysphere[0][0], point[1]-mysphere[0][1],point[2]-mysphere[0][2]])
i = normalize([light[0]-point[0], light[1]-point[1],light[2]-point[2]])
costheta = dot(n,i)
#if(costheta<0):
# costheta=0
#color = int(costheta*255)
#print n[0]
#print costheta
#print color
im.putpixel((x,y),(int(-n[0]*255),int(-n[1]*255),int(-n[2]*255)))
#im.putpixel((x,y),(255,255,0))
#im.show()
im.save("sphr_"+str(mysphere[1])+".png","PNG")
File Handling Dandelion
import os
import re
def listdirs(folder):
return [d for d in os.listdir(folder) if os.path.isdir(os.path.join(folder, d))]
paths = 'N:/01_OUT'
pattern = 'GB\d+_SC\d+.*_T\d+'
#text = 'GB45_SC34_T3'
#match = re.search(pattern, text)
#print match
f = open('N:/01_OUT/summary.html', 'w')
#f.write('0123456789abcdef')
count = 0
for dir in listdirs(paths):
f.write("<hr>\n\n</br></br>"+dir+"</br>")
subdir = listdirs(paths+"/"+dir)
for takes in subdir:
tk = listdirs(paths+"/"+dir+"/"+takes)
#if(len(tk)>4):
#f.write("\n"+paths+"/"+dir+"/"+takes)
f.write("\n"+paths+"/"+dir+"/"+takes+"</br>")
for take in tk:
match = re.search(pattern,take)
if(match != "None"):
count+=1
if(count % 2 == 1):
c = "#fbfbfb";
else:
c = "#eeeeee";
f.write("\n<div style='background-color:"+c+";'> <input type=checkbox name=\""+(paths+"/"+dir+"/"+takes)+"\" CHECKED>"+take+"</div>")
print take
#print takes+": "+str(len(tk))
#print take+" ("+str(len(tk))+")"
f.close();
raw_input("-")
#for dir in os.listdir(path):
# for subdir in os.listdir(path+"/"+dir):
# takes = os.listdir(path+"/"+dir+"/"+subdir)
# directories=[d for d in os.listdir(path+"/"+dir+"/"+subdir) if os.path.isdir(d)]
# #print subdir+":"+str(len(takes))
# print directories
#
#raw_input("Press ENTER to exit")