It is example code to scrape it:
furas.pl#!/usr/bin/env python3
# date: 2020.02.23
# https://stackoverflow.com/questions/60362610/python-selenium-click-a-button/
import selenium.webdriver
url = 'http://cafe.daum.net/WekiMeki'
driver = selenium.webdriver.Chrome()
#driver = selenium.webdriver.Firefox()
driver.get(url)
frame = driver.find_element_by_id('down')
driver.switch_to.frame(frame)
driver …It is example code to scrape it:
#
# https://stackoverflow.com/a/47933667/1832058
#
from bs4 import BeautifulSoup
import requests
params = {
'zip': '03062',
'address': 'Nashua,+NH',
'latitude': "42.73040008544922",
'longitude': '-71.49479675292969',
'distance': 50000,
'selectedEntity': 'c24578',
'entitySelectingHelper.selectedEntity2': 'c25202',
'minPrice': '',
'maxPrice': '',
'minMileage': '',
'maxMileage': '',
'transmission': 'ANY',
'bodyTypeGroup': '',
'serviceProvider': '',
'page': 1,
'filterBySourcesString': '',
'filterFeaturedBySourcesString …It is example code to scrape it:
#!/usr/bin/env python3
#
# https://stackoverflow.com/a/47888293/1832058
#
import scrapy
data = '''https://www.ceneo.pl/48523541, 1362
https://www.ceneo.pl/46374217, 2457'''
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://www.ceneo.pl/33022301']
def start_requests(self):
# get data from …It is example code to scrape it:
#!/usr/bin/env python3
import scrapy
#from scrapy.commands.view import open_in_browser
#import json
class FileDownloaderItem(scrapy.Item):
file_urls = scrapy.Field()
files = scrapy.Field()
full_urls = scrapy.Field()
class MySpider(scrapy.Spider):
name = 'myspider'
allowed_domains = ['www.cgtrader.com']
start_urls = ['https://www.cgtrader.com/free-3d-print-models …It is example code to scrape it:
import selenium.webdriver
url = "https://www.claytoncountyga.gov/government/sheriff/inmate-search"
driver = selenium.webdriver.Firefox()
driver.get(url)
iframes = driver.find_elements_by_tag_name('iframe')
print('iframes:', iframes)
driver.switch_to.frame(iframes[0])
item = driver.find_element_by_id('name')
print('name:', item)
item.send_keys("John")
item = driver.find_element_by_name …It is example code to scrape it:
#
# https://stackoverflow.com/a/47744797/1832058
#
from bs4 import BeautifulSoup
import requests
html = requests.get("https://www.cnbc.com/2017/12/07/pinterest-hires-former-facebook-exec-gary-johnson-to-run-corporate-dev.html").text
soup = BeautifulSoup(html, 'html5lib')
all_paragraphs = soup.find_all('p')
for p in all_paragraphs:
#print(p) # all HTML
print(p …It is example code to scrape it:
# author: https://blog.furas.pl
# date: 2020.08.04
# link: https://stackoverflow.com/questions/63246707/python-scraping-create-payload-cnmv-es-and-render-javascript/
import requests
from bs4 import BeautifulSoup
url = 'https://www.cnmv.es/portal/Consultas/BusquedaPorEntidad.aspx' # '?lang=en'
search_text = 'aaa' # 'abc'
r = requests.get(url)
#print(response.text …It is example code to scrape it:
#!/usr/bin/env python3
#
# https://stackoverflow.com/a/47940659/1832058
#
from bs4 import BeautifulSoup
import requests
link = "http://www.cnnvd.org.cn/web/vulnerability/querylist.tag"
req = requests.get(link)
web = req.text
soup = BeautifulSoup(web, "lxml")
cve_name = []
cve_link = []
for par_ in soup …It is example code to scrape it:
#!/usr/bin/env python3
# date: 2019.12.02
# https://stackoverflow.com/questions/59132449/what-is-the-proper-syntax-for-find-in-bs4
import requests
from bs4 import BeautifulSoup
url = 'https://www.coinbase.com/charts'
r = requests.get(url, headers=headers)
soup = BeautifulSoup(r.text, 'html.parser')
all_tr = soup.find_all('tr')
data …It is example code to scrape it:
import requests
import datetime
import csv
start_date = '2016.01.01'
finish_date = '2017.01.01'
start_date = datetime.datetime.strptime(start_date, '%Y.%m.%d')
finish_date = datetime.datetime.strptime(finish_date, '%Y.%m.%d')
start_timestamp = int(start_date.timestamp() * 1000)
one_day = datetime.timedelta(days=1)
finish_timestamp = int(finish_date …It is example code to scrape it:
# date: 2019.05.09
# author: Bartłomiej 'furas' Burek
# https://stackoverflow.com/questions/56059703/how-can-i-make-lxml-save-two-pages-to-the-pages-so-it-can-be-read-by-the-tree
from lxml import html
import requests
data = {
'BTC': 'id-bitcoin',
'TRX': 'id-tron',
# ...
'HC': 'id-hypercash',
'XZC': 'id-zcoin',
}
all_results = {}
for url in ('https://coinmarketcap.com/', 'https://coinmarketcap.com/2'):
page = requests.get …It is example code to scrape it:
# author: https://blog.furas.pl
# date: 2020.07.25
# link: https://stackoverflow.com/questions/63075215/read-html-where-required-table-needs-users-input/
import pandas as pd
all_dfs = pd.read_html('https://coinmarketcap.com/exchanges/bitfinex/')
df = all_dfs[2]
df[ df['Pair'].str.endswith('USD') ]
It is example code to scrape it:
#!/usr/bin/env python3
#
# https://stackoverflow.com/a/47729218/1832058
#
import scrapy
class CollegiateSpider(scrapy.Spider):
name = 'Collegiate'
allowed_domains = ['collegiate-ac.com']
start_urls = ['https://collegiate-ac.com/uk-student-accommodation/']
# Step 1 - Get the area links
def parse(self, response):
for url in response.xpath('//*[@id="top …It is example code to scrape it:
#!/usr/bin/env python3
# date: 2019.08.06
# https://stackoverflow.com/questions/57366488/how-to-pass-the-single-link-in-a-nested-url-scrape
import scrapy
def clean(text):
text = text.replace('\xa0', ' ')
text = text.strip().split('\n')
text = ' '.join(x.strip() for x in text)
return text
class PaniniSpider(scrapy.Spider):
name …It is example code to scrape it:
#!/usr/bin/env python3
# date: 2019.11.24
# https://stackoverflow.com/questions/59019810/python-web-scraping-ahref-link-and-articles-not-showing-up-in-source-code
import selenium.webdriver
url = 'https://corporate.dow.com/en-us/news.html'
driver = selenium.webdriver.Firefox()
driver.get(url)
all_items = driver.find_elements_by_xpath('//ul[@class="results__list"]/li')
for item in all_items …It is example code to scrape it:
#!/usr/bin/env python3
#
# https://stackoverflow.com/a/48017689/1832058
#
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://www.coursetalk.com/subjects/data-science/courses']
def parse(self, response):
print('url:', response.url)
for item in response.xpath('.//*[@class="as-table-cell"]/a/@href …It is example code to scrape it:
#!/usr/bin/env python3
#
# https://stackoverflow.com/a/47720827/1832058
#
import requests
from bs4 import BeautifulSoup
import csv
filename = "output.csv"
f = open(filename, 'w', newline="", encoding='utf-8')
csvwriter = csv.writer(f)
csvwriter.writerow( ["Date", "Location", "Title", "Price"] )
offset = 0
while True:
print …It is example code to scrape it:
#!/usr/bin/env python3
# date: 2019.12.18
# https://stackoverflow.com/questions/59386434/selenium-webdriver-i-want-to-click-on-the-next-page-till-last-page/59387563#59387563
from selenium import webdriver
#from bs4 import BeautifulSoup as bs
import time
url = 'https://curecity.in/vendor-list.php?category=Doctor&filters_location=Jaipur&filters%5Bsubareas_global%5D=&filters_speciality='
#driver …It is example code to scrape it:
#
# https://api.data.gov/
# https://regulationsgov.github.io/developers/basics/
#
# https://stackoverflow.com/a/48030949/1832058
#
import requests
import json
import time
all_titles = ['EPA-HQ-OAR-2013-0602']
api_key = 'PB36zotwgisM02kED1vWwvf7BklqCObDGVoyssVE'
api_base='https://api.data.gov/regulations/v3/'
api_url = '{}docket.json?api_key={}&docketId='.format(api_base, api_key)
try:
for …It is example code to scrape it:
import requests
from bs4 import BeautifulSoup
import json
base_url = 'https://www.deezer.com/en/profile/1589856782/loved'
r = requests.get(base_url)
soup = BeautifulSoup(r.text, 'html.parser')
all_scripts = soup.find_all('script')
data = json.loads(all_scripts[6].get_text()[27:])
print('key:', data.keys())
print …