selenium

driver = webdriver.Firefox(options=firefox_options,
                           executable_path='/depot/datamine/bin/geckodriver')
driver.get("https://datamine.purdue.edu")
print(driver.page_source[:500])

driver = webdriver.Firefox(options=firefox_options,
                           executable_path='/depot/datamine/bin/geckodriver')
driver.get("https://datamine.purdue.edu")
my_element = driver.find_element("xpath", "//section[@class='office__hero']")
print(my_element.get_attribute("outerHTML"))

driver = webdriver.Firefox(options=firefox_options,
                           executable_path='/depot/datamine/bin/geckodriver')
driver.get("https://datamine.purdue.edu")
my_element = driver.find_element("xpath", "//section[@class='office__hero']")
print(my_element.get_attribute("innerHTML"))

from selenium.webdriver.common.keys import Keys
import time

driver = webdriver.Firefox(options=firefox_options,
                           executable_path='/depot/datamine/bin/geckodriver')

# get the webpage
driver.get("https://www.purdue.edu/directory")

# isolate the search bar "input" element
element = driver.find_element("xpath", "//input[@id='basicSearchInput']")

# use "send_keys" to type in the search bar
element.send_keys("mdw")

# just like when you use a browser, you either need to push "enter" or click on the search button. This time, we will press enter.
# Note that this is where we needed the import statement
element.send_keys(Keys.RETURN)

# We can delay the program to allow the page to load
time.sleep(5)

# get the table(s)
elements = driver.find_elements("xpath", "//table[@class='more']")

# how many tables are there?
print(len(elements))

# comment out the Keys.RETURN command
# element.send_keys(Keys.RETURN)

# find the button to execute the search
button = driver.find_element("xpath", "//a[@id='glass']")

# click the button
button.click()

<table class="more">
    <thead>
        <tr>
            <th scope="col" colspan="2">mark daniel ward</th>
        </tr>
    </thead>
    <tbody>
        <tr>
            <th class="icon-key" scope="row">Alias</th>
            <td>mdw</td>
        </tr>
        <tr>
            <th class="icon-envelope-alt">Email</th>
            <td><a href="mailto:[email protected]">[email protected]</a></td>
        </tr>
        <tr>
            <th class="icon-library" scope="row">Campus</th>
            <td>west lafayette</td>
        </tr>
        <tr>
            <th class="icon-sitemap">Department</th>
            <td>statistics</td>
        </tr>
        <tr>
            <th class="icon-briefcase" scope="row">Title</th>
            <td>professor of statistics</td>
        </tr>
    </tbody>
</table>

# note since we used the plural `find_elements`, elements is a list.
# If we used the singular `find_element`, we wouldn't need the [0] part because we wouldn't have a list
elements[0].get_attribute("outerHTML")

# first get the name using .// which searches starting in the current element
# if we used //, it would search the entire webpage, not just our element
name = elements[0].find_element("xpath", ".//thead/tr/th").text
print(name)

# next, get the alias. The xpath expression first gets the "th" element with class=icon-key. We want the content of the following td element, and since the next "td" element is at the same level of nesting as the "th" element, it is referred to as a "sibling"
# following-sibling::td finds the "td" sibling immediately following the current "th" element
alias = elements[0].find_element("xpath", ".//th[@class='icon-key']/following-sibling::td").text
print(alias)

# next, get the email. If you don't specify what the attribute is equal to, it will evaluate to true if there is any value, and false otherwise.
email = elements[0].find_element("xpath", ".//a[@href]").text
print(email)

# next, get the campus
campus = elements[0].find_element("xpath", ".//th[@class='icon-sitemap']/following-sibling::td").text
print(campus)

# finally, get the title
title = elements[0].find_element("xpath", ".//th[@class='icon-briefcase']/following-sibling::td").text
print(title)

import requests

response = requests.get('https://www.shutterstock.com/search/dog+side+view')
print(response.text[:500])

import lxml.html

tree = lxml.html.fromstring(response.text)
elements = tree.xpath("//img[@data-automation='mosaic-grid-cell-image']")
print(len(elements))

my_headers = {'User-Agent': 'Mozilla/5.0'}
html = requests.get('https://www.shutterstock.com/search/dog+side+view',
                    headers=my_headers)

for element in elements:
    print(element.attrib.get("src"))

driver.get("https://www.shutterstock.com/search/dog+side+view")

# create a scroll function that emulates scrolling
import time
def scroll(driver, scroll_point):
    driver.execute_script(f'window.scrollTo(0, {scroll_point});')
    time.sleep(5)

# Needed to get the window size set right
height = driver.execute_script('return document.body.scrollHeight')
driver.set_window_size(900,height+100)

# begin scrolling a bit, 1/4 of the page at a time, maybe
scroll(driver, height/4)
scroll(driver, height*2/4)
scroll(driver, height*3/4)
scroll(driver, height)

# extract the image links
elements = driver.find_elements("xpath", "//img[@data-automation='mosaic-grid-cell-image']")
for element in elements:
  print(element.get_attribute("src"))

import os
from urllib.parse import urlparse

def get_filename_from_url(url: str) -> str:
    """
    Given a link to a file, return the filename with extension.
    Args:
        url (str): The url of the file.
    Returns:
        str: A string with the filename, including the file extension.
    """
    return os.path.basename(urlparse(url).path)

import requests
from pathlib import Path
import getpass

def scrape_image(from_url: str, to_dir: str = f'/home/{getpass.getuser()}'):
    """
    Given a url to an image, scrape the image and save the image to the provided directory.
    If no directory is provided, by default, save to the user's home directory.
    Args:
        from_url (str): U
        to_dir (str, optional): [description]. Defaults to f'/home/{getpass.getuser()}'.
    """
    resp = requests.get(from_url)

    # this function is from the previous example
    filename = get_filename_from_url(from_url)

    # Make directory if doesn't already exist
    Path(to_dir).mkdir(parents=True, exist_ok=True)

    file = open(f'{to_dir}/{filename}', "wb")
    file.write(resp.content)
    file.close()

for element in elements:
    scrape_image(element.get_attribute("src"))