Data Collection Bot with Python and Selenium

Overview

In this article, we will:

  • Firefox
  • Python 3
  • Selenium
  • Gecko Webdriver
  • Notepad ++
  • Write a class for our “Bot”
  • Get_crypto_data function within our class
  • Write_file function for the class to write the real time data to a local cloud file.

Installation

Open up Internet Explorer to navigate to mozilla.org. You may need to add some exceptions to the built in Windows firewall to enable the Firefox installer to download.

python --version
pip install selenium
setx path "%path%;c:\Users\Administrator\Desktop\geckodriver.exe"

Python Script

Now let’s decide what we want our Python Selenium “Web Bot” to do. Well, it would be cool if our “bot” would grab some crypto currency market data for us. Write that data to a local file. Then we can plug that data into any other Alibaba cloud service. Get ready to type!

from selenium import webdriver
import datetime
import time
browser = webdriver.Firefox()
browser.get("http://www.baidu.com/")
from selenium import webdriverclass Bot():
def __init__(self,url):
self.browser = webdriver.Firefox()
self.url = url
def get_web_page(self):
self.browser.get(self.url)
bot = Bot("http://www.baidu.com/")
bot.get_web_page()
from selenium import webdriver
class Bot():
def __init__(self,crypto):
self.browser = webdriver.Firefox()
### CRYPTO TO SEARCH FOR
self.crypto = crypto
def get_crypto_data(self):
self.browser.get("https://www.coinwatch.com/")
### FIND SEARCH FORM AND IMPUTE KEY WORD
print("FINDING SEARCH FORM")
element = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/header/section[3]/div/div/div/div[1]/div/input")
element.clear()
element.send_keys(self.crypto)
element = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/header/section[3]/div/div/div/div[1]/div/div/button")
element.click()
bot = Bot("BTC")
bot.get_crypto_data()
from selenium import webdriver
import datetime
import time
class Bot():
def __init__(self,crypto):
self.browser = webdriver.Firefox()
### CRYPTO TO SEARCH FOR
self.crypto = crypto
self.price = None
self.cap = None
self.volume_24h = None
self.change_24h = None
def get_crypto_data(self):
self.browser.get("https://www.coinwatch.com/")
### FIND SEARCH FORM AND IMPUTE KEY WORD
print("FINDING SEARCH FORM")
elem = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/header/section[3]/div/div/div/div[1]/div/input")
elem.clear()
elem.click()
elem.send_keys(self.crypto)
elem = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/header/section[3]/div/div/div/div[1]/div/div/button")
time.sleep(1)
elem.click()
### GET MARKET DATA
print("GETTING MARKET DATA")
time.sleep(5)
self.price = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/main/div/div/article/div[1]/div/div[3]/div/div/span").text
self.cap = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/main/div/div/article/div[2]/div/main/div[1]/section/div[1]/div/div[2]/div/div[1]/span[2]").text
self.volume_24h = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/main/div/div/article/div[2]/div/main/div[1]/section/div[1]/div/div[2]/div/div[2]/span[2]").text
self.change_24h = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/main/div/div/article/div[2]/div/main/div[1]/section/div[1]/div/div[2]/div/div[8]/span[2]").text
print("PRINTING MARKET DATA")
print("PRICE " + str(self.price))
print("CAP " + str(self.cap))
print("24H VOLUME " + str(self.volume_24h))
print("24H CHANGE " + str(self.change_24h))
self.browser.close()
return
bot = Bot("BTC")
bot.get_crypto_data()
def write_file(self):
save_file = open(self.crypto + "_market.txt","a")
time_stamp = str(datetime.now())
save_file.write("\n" + time_stamp + "\n")
save_file.write("\nPRICE: " + str(self.price))
save_file.write("\nCAP: " + str(self.cap))
save_file.write("\n24H VOLUME: " + str(self.volume_24h))
save_file.write("\n24H CHANGE: " + str(self.change_24h))
btc_bot = Bot("BTC")
eth_bot = Bot("ETH")
btc_bot.get_crypto_data()
btc_bot.write_file()
eth_bot.get_crypto_data()
eth_bot.write_file()
from selenium import webdriver
from datetime import datetime
import time
class Bot(): def __init__(self,crypto):
self.browser = webdriver.Firefox()
### CRYPTO TO SEARCH FOR
self.crypto = crypto
self.price = None
self.cap = None
self.volume_24h = None
self.change_24h = None
def get_crypto_data(self):
self.browser.get("https://www.coinwatch.com/")
### FIND SEARCH FORM AND IMPUT KEY WORD
print("FINDING SEARCH FORM")
elem = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/header/section[3]/div/div/div/div[1]/div/input")
elem.clear()
elem.click()
elem.send_keys(self.crypto)
elem = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/header/section[3]/div/div/div/div[1]/div/div/button")
time.sleep(1)
elem.click()
### GET MARKET DATA
print("GETTING MARKET DATA")
time.sleep(5)
self.price = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/main/div/div/article/div[1]/div/div[3]/div/div/span").text
self.cap = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/main/div/div/article/div[2]/div/main/div[1]/section/div[1]/div/div[2]/div/div[1]/span[2]").text
self.volume_24h = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/main/div/div/article/div[2]/div/main/div[1]/section/div[1]/div/div[2]/div/div[2]/span[2]").text
self.change_24h = self.browser.find_element_by_xpath("//html/body/div[1]/div/div/div/div/main/div/div/article/div[2]/div/main/div[1]/section/div[1]/div/div[2]/div/div[8]/span[2]").text
print("PRINTING MARKET DATA")
print("PRICE " + str(self.price))
print("CAP " + str(self.cap))
print("24H VOLUME " + str(self.volume_24h))
print("24H CHANGE " + str(self.change_24h))
self.browser.close()
return
def write_file(self): time_stamp = str(datetime.now()) save_file = open(self.crypto + "_market.txt","a")
save_file.write(self.crypto)
save_file.write("\n" + time_stamp + "\n")
save_file.write("\nPRICE: " + str(self.price))
save_file.write("\nCAP: " + str(self.cap))
save_file.write("\n24H VOLUME: " + str(self.volume_24h))
save_file.write("\n24H CHANGE: " + str(self.change_24h) + "\n")
btc_bot = Bot("BTC")
eth_bot = Bot("ETH")
btc_bot.get_crypto_data()
btc_bot.write_file()
eth_bot.get_crypto_data()
eth_bot.write_file()

Original Source

https://www.alibabacloud.com/blog/data-collection-bot-with-python-and-selenium_594948?spm=a2c41.13076703.0.0

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Alibaba Cloud

Alibaba Cloud

Follow me to keep abreast with the latest technology news, industry insights, and developer trends. Alibaba Cloud website:https://www.alibabacloud.com