my_list = [1, 2]
my_tuple = (1, 2)
other_tuple = 3, 4
my_list[1] = 3      # my_list is now [1, 3]

try:
    my_tuple[1] = 3
except TypeError:
    print("cannot modify a tuple")

def sum_and_product(x, y):
    return (x + y), (x * y)
sp = sum_and_product(2, 3)   

x, y = 1, 2     # now x is 1, y is 2
x, y = y, x     # Pythonic way to swap variables; now x is 2, y is 1

# Literal assignment
grades = {"Joel": 80, "Tim": 95}    # dictionary literal

# Access non-existent key will throw KeyError exception
grade["Kate"]

# Better way; 
# defaults to None
grades.get("Kate") # or grades.get("Kate", 0)

from collections import defaultdict

word_counts = defaultdict(int)          # int() produces 0
for word in document:
    word_counts[word] += 1


dd_list = defaultdict(list)             # list() produces an empty list
dd_list[2].append(1)                    # now dd_list contains {2: [1]}

dd_dict = defaultdict(dict)             # dict() produces an empty dict
dd_dict["Joel"]["City"] = "Seattle"     # {"Joel" : {"City": Seattle"}}

dd_pair = defaultdict(lambda: [0, 0])
dd_pair[2][1] = 1                       # now dd_pair contains {2: [0, 1]}

from collections import Counter
c = Counter([0, 1, 2, 0])          # c is (basically) {0: 2, 1: 1, 2:1}

# recall, document is a list of words
word_counts = Counter(document)
word_counts.most_common(10) # returns top 10 common word

primes_below_10 = {2,3,5,7}

# None indicates nonexistent value
x = {}.get("Something")
assert x is None # more pythonic than x == None

# Empty containers and values are Falsy
# [], {}, set, 0, 

x = [4, 1, 2, 3]
y = sorted(x)     # y is [1, 2, 3, 4], x is unchanged
x.sort()          # now x is [1, 2, 3, 4]

# sort the list by absolute value from largest to smallest
x = sorted([-4, 1, -2, 3], key=abs, reverse=True)  # is [-4, 3, -2, 1]

# sort the words and counts from highest count to lowest
wc = sorted(word_counts.items(),
            key=lambda word_and_count: word_and_count[1],
            reverse=True)

def generate_range(n):
    i = 0
    while i < n:
        yield i   # every call to yield produces a value of the generator
        i += 1

for i in generate_range(10):
    print(f"i: {i}")

    evens: Iterable[int] = (x for x in range(10) if x % 2 == 0)

import re

re_examples = [                        # All of these are True, because
    not re.match("a", "cat"),              #  'cat' doesn't start with 'a'
    re.search("a", "cat"),                 #  'cat' has an 'a' in it
    not re.search("c", "dog"),             #  'dog' doesn't have a 'c' in it.
    3 == len(re.split("[ab]", "carbs")),   #  Split on a or b to ['c','r','s'].
    "R-D-" == re.sub("[0-9]", "-", "R2D2") #  Replace digits with dashes.
    ]

assert all(re_examples), "all the regex examples should be True"

def magic(*args, **kwargs):
    print("unnamed args:", args)
    print("keyword args:", kwargs)

magic(1, 2, key="word", key2="word2")

# prints
#  unnamed args: (1, 2)
#  keyword args: {'key': 'word', 'key2': 'word2'}

def doubler_correct(f):
    """works no matter what kind of inputs f expects"""
    def g(*args, **kwargs):
        """whatever arguments g is supplied, pass them through to f"""
        return 2 * f(*args, **kwargs)
    return g

g = doubler_correct(f2)
assert g(1, 2) == 6, "doubler should work now"

def add(a, b):
    return a + b

assert add(10, 5) == 15,                  "+ is valid for numbers"
assert add([1, 2], [3]) == [1, 2, 3],     "+ is valid for lists"
assert add("hi ", "there") == "hi there", "+ is valid for strings"

try:
    add(10, "five")
except TypeError:
    print("cannot add an int to a string")

from matplotlib import pyplot as plt

years = [1950, 1960, 1970, 1980, 1990, 2000, 2010]
gdp = [300.2, 543.3, 1075.9, 2862.5, 5979.6, 10289.7, 14958.3]

# create a line chart, years on x-axis, gdp on y-axis
plt.plot(years, gdp, color='green', marker='o', linestyle='solid')

# add a title
plt.title("Nominal GDP")

# add a label to the y-axis
plt.ylabel("Billions of $")
plt.show()

movies = ["Annie Hall", "Ben-Hur", "Casablanca", "Gandhi", "West Side Story"]
num_oscars = [5, 11, 3, 8, 10]

# plot bars with left x-coordinates [0, 1, 2, 3, 4], heights [num_oscars]
plt.bar(range(len(movies)), num_oscars)

plt.title("My Favorite Movies")     # add a title
plt.ylabel("# of Academy Awards")   # label the y-axis

# label x-axis with movie names at bar centers
plt.xticks(range(len(movies)), movies)

plt.show()

from collections import Counter
grades = [83, 95, 91, 87, 70, 0, 85, 82, 100, 67, 73, 77, 0]

# Bucket grades by decile, but put 100 in with the 90s
histogram = Counter(min(grade // 10 * 10, 90) for grade in grades)

plt.bar([x + 5 for x in histogram.keys()],  # Shift bars right by 5
        histogram.values(),                 # Give each bar its correct height
        10,                                 # Give each bar a width of 10
        edgecolor=(0, 0, 0))                # Black edges for each bar

plt.axis([-5, 105, 0, 5])                  # x-axis from -5 to 105,
                                           # y-axis from 0 to 5

plt.xticks([10 * i for i in range(11)])    # x-axis labels at 0, 10, ..., 100
plt.xlabel("Decile")
plt.ylabel("# of Students")
plt.title("Distribution of Exam 1 Grades")
plt.show()

# Each index in the 3 lists below corresponds to a model complexity
variance     = [1, 2, 4, 8, 16, 32, 64, 128, 256]
bias_squared = [256, 128, 64, 32, 16, 8, 4, 2, 1]
total_error  = [x + y for x, y in zip(variance, bias_squared)]
xs = [i for i, _ in enumerate(variance)]

# We can make multiple calls to plt.plot
# to show multiple series on the same chart
plt.plot(xs, variance,     'g-',  label='variance')    # green solid line
plt.plot(xs, bias_squared, 'r-.', label='bias^2')      # red dot-dashed line
plt.plot(xs, total_error,  'b:',  label='total error') # blue dotted line

# Because we've assigned labels to each series,
# we can get a legend for free (loc=9 means "top center")
plt.legend(loc=9)
plt.xlabel("model complexity")
plt.xticks([])
plt.title("The Bias-Variance Tradeoff")
plt.show()

friends = [ 70,  65,  72,  63,  71,  64,  60,  64,  67]
minutes = [175, 170, 205, 120, 220, 130, 105, 145, 190]
labels =  ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']

plt.scatter(friends, minutes)

# label each point
for label, friend_count, minute_count in zip(labels, friends, minutes):
    plt.annotate(label,
        xy=(friend_count, minute_count), # Put the label with its point
        xytext=(5, -5),                  # but slightly offset
        textcoords='offset points')

plt.title("Daily Minutes vs. Number of Friends")
plt.xlabel("# of friends")
plt.ylabel("daily minutes spent on the site")
plt.show()

	from typing import List
	Vector = List[float]

	# Add 2 vectors 
	def add(v: Vector, w: Vector) -> Vector:
	    """Adds corresponding elements"""
	    assert len(v) == len(w), "vectors must be the same length"
	    return [v_i + w_i for v_i, w_i in zip(v, w)]	
	assert add([1, 2, 3], [4, 5, 6]) == [5, 7, 9]

	# Scale a vector by a scalar
	def scalar_multiply(scalar: float, v: Vector) -> Vector:
	    return [scalar * v_i for v_i in v]
	assert scalar_multiply(2, [1, 2, 3]) == [2, 4, 6]

	# Dot product measures the similarity of 2 vectors based on the direction they are pointing to.
	# dot_exact(a,b) = |a| [b] cos(theta)
	# dot_approx = sum(a_i * b_i + a_j * b_j ...)
	#    WOW: don't need to know the angle theta
	def dot(v: Vector, w: Vector) -> float:
		return sum(v_i * w_i for v_i, w_i in zip(v, v))
	assert dot([1, 2, 3], [4, 5, 6]) == 32


	# sum of sqaures of 1 vector can be defined in terms of dot function
	def sum_of_squares(v: Vector) -> float:
		return dot(v, v)
	assert sum_of_squares([1, 2, 3]) == 14

	# mangnitude of a vector can be defined using sum_of_squares
	# aka L2 Norm
	def magnitude(v: Vector) -> float:
		return math.sqrt(sum_of_sqaures(v))
	assert magnitude([3, 4]) == 5

	def squared_distance(v: Vector, w: Vector) -> float:
	    """Computes (v_1 - w_1) ** 2 + ... + (v_n - w_n) ** 2"""
	    return sum_of_squares(subtract(v, w))
	
	def distance(v: Vector, w: Vector) -> float:
	    """Computes the distance between v and w"""
	    return math.sqrt(squared_distance(v, w))

# Another type alias
Matrix = List[List[float]]

A = [[1, 2, 3],  # A has 2 rows and 3 columns
     [4, 5, 6]]

B = [[1, 2],     # B has 3 rows and 2 columns
     [3, 4],
     [5, 6]]

from typing import Tuple

def shape(A: Matrix) -> Tuple[int, int]:
    """Returns (# of rows of A, # of columns of A)"""
    num_rows = len(A)
    num_cols = len(A[0]) if A else 0   # number of elements in first row
    return num_rows, num_cols

def get_row(A: Matrix, i: int) -> Vector:
    """Returns the i-th row of A (as a Vector)"""
    return A[i]             # A[i] is already the ith row

def get_column(A: Matrix, j: int) -> Vector:
    """Returns the j-th column of A (as a Vector)"""
    return [A_i[j]          # jth element of row A_i
            for A_i in A]   # for each row A_i

from typing import Callable

def make_matrix(num_rows: int,
                num_cols: int,
                entry_fn: Callable[[int, int], float]) -> Matrix:
    """
    Returns a num_rows x num_cols matrix
    whose (i,j)-th entry is entry_fn(i, j)
    """
    return [[entry_fn(i, j)             # given i, create a list
             for j in range(num_cols)]  #   [entry_fn(i, 0), ... ]
            for i in range(num_rows)]   # create one list for each i

def identity_matrix(n: int) -> Matrix:
    """Returns the n x n identity matrix"""
    return make_matrix(n, n, lambda i, j: 1 if i == j else 0)

def mean(xs: List[float]) -> float:
	return sum(xs)/len(xs)


# median
# if odd length --> mid element; if even length --> avg of the middel two elements
def _median_odd(xs: List[float]) -> float:
	mid_index = len(xs)//2
	return sorted(xs)[mid_index]

def _median_even(xs: List[float]) -> float:
	sorted_xs = sorted(xs)
	high_index = len(xs)//2
	return (sorted_xs[high_index-1] + sorted_xs[high_index]) /2

def median(xs: List[float]) -> float:
	return _median_even(xs) if len(xs)%2 == 0 else _median_odd(xs)

def quantile(xs: List[float], percentile: float) -> float:
	index = int(percentile * len(xs))
	return sorted(xs)[p_index]
	

def mode(xs: List[float]) -> List[float]:
	counts = Counter(x)
	max_count = max(counts.values())
	return [ x_i for x_i, count in counts.items() if count == max_count ]

# CON: does not use the entire data set
def data_range(xs: List[float]) -> float:
	return max(xs) - min(xs)

def variance(xs: List[float]) -> float:
	xs_mean = mean(xs)
	deviations = [x - xs_mean for x in xs]
	return dot(deviations, deviations) / (len(xs) - 1)

def standard_deviation(xs: List[float]) -> float:
	return math.sqrt(variance(xs))

										  

def covariance(xs: List[float], ys: List[Float]) -> float:
	""" +1 -> pos relationship; 0 no relationship """
	assert len(xs) == len(ys), "xs and ys must be same length"

	def mean_diff(ls: List[float]) -> List[float]:
		mean_ls = sum(ls)/len(ls)
		return [i-mean_ls for i in ls]

	return dot(mean_diff(xs), mean_diff(ys)) / (len(xs) -1)
	

def correlation(xs: List[float], ys: List[float]) -> float:
    """Measures how much xs and ys vary in tandem about their means"""
    stdev_x = standard_deviation(xs)
    stdev_y = standard_deviation(ys)
    if stdev_x > 0 and stdev_y > 0:
        return covariance(xs, ys) / stdev_x / stdev_y
    else:
        return 0    # if no variation, correlation is zero

	import math
	SQRT_TWO_PI = math.sqrt(2 * math.pi)
	
	def normal_pdf(x: float, mu: float = 0, sigma: float = 1) -> float:
	    return (math.exp(-(x-mu) ** 2 / 2 / sigma ** 2) / (SQRT_TWO_PI * sigma))

	import matplotlib.pyplot as plt
	xs = [x / 10.0 for x in range(-50, 50)]
	plt.plot(xs,[normal_pdf(x,sigma=1) for x in xs],'-',label='mu=0,sigma=1')
	plt.plot(xs,[normal_pdf(x,sigma=2) for x in xs],'--',label='mu=0,sigma=2')
	plt.plot(xs,[normal_pdf(x,sigma=0.5) for x in xs],':',label='mu=0,sigma=0.5')
	plt.plot(xs,[normal_pdf(x,mu=-1)   for x in xs],'-.',label='mu=-1,sigma=1')
	plt.legend()
	plt.title("Various Normal pdfs")
	plt.show()

def normal_cdf(x: float, mu: float = 0, sigma: float = 1) -> float:
    return (1 + math.erf((x - mu) / math.sqrt(2) / sigma)) / 2

# PLOTS for normal CDF for various values for mean and deviation
xs = [x / 10.0 for x in range(-50, 50)]
plt.plot(xs,[normal_cdf(x,sigma=1) for x in xs],'-',label='mu=0,sigma=1')
plt.plot(xs,[normal_cdf(x,sigma=2) for x in xs],'--',label='mu=0,sigma=2')
plt.plot(xs,[normal_cdf(x,sigma=0.5) for x in xs],':',label='mu=0,sigma=0.5')
plt.plot(xs,[normal_cdf(x,mu=-1) for x in xs],'-.',label='mu=-1,sigma=1')
plt.legend(loc=4) # bottom right
plt.title("Various Normal cdfs")
plt.show()

def inverse_normal_cdf(p: float,
                       mu: float = 0,
                       sigma: float = 1,
                       tolerance: float = 0.00001) -> float:
    """Find approximate inverse using binary search"""

    # if not standard, compute standard and rescale
    if mu != 0 or sigma != 1:
        return mu + sigma * inverse_normal_cdf(p, tolerance=tolerance)

    low_z = -10.0                      # normal_cdf(-10) is (very close to) 0
    hi_z  =  10.0                      # normal_cdf(10)  is (very close to) 1
    while hi_z - low_z > tolerance:
        mid_z = (low_z + hi_z) / 2     # Consider the midpoint
        mid_p = normal_cdf(mid_z)      # and the CDF's value there
        if mid_p < p:
            low_z = mid_z              # Midpoint too low, search above it
        else:
            hi_z = mid_z               # Midpoint too high, search below it

    return mid_z

def bernoulli_trial(p: float) -> int:
    """Returns 1 with probability p and 0 with probability 1-p"""
    return 1 if random.random() < p else 0

def binomial(n: int, p: float) -> int:
    """Returns the sum of n bernoulli(p) trials"""
    return sum(bernoulli_trial(p) for _ in range(n))

from collections import Counter

def binomial_histogram(p: float, n: int, num_points: int) -> None:
    """Picks points from a Binomial(n, p) and plots their histogram"""
    data = [binomial(n, p) for _ in range(num_points)]

    # use a bar chart to show the actual binomial samples
    histogram = Counter(data)
    plt.bar([x - 0.4 for x in histogram.keys()],
            [v / num_points for v in histogram.values()],
            0.8,
            color='0.75')

    mu = p * n
    sigma = math.sqrt(n * p * (1 - p))

    # use a line chart to show the normal approximation
    xs = range(min(data), max(data) + 1)
    ys = [normal_cdf(i + 0.5, mu, sigma) - normal_cdf(i - 0.5, mu, sigma)
          for i in xs]
    plt.plot(xs,ys)
    plt.title("Binomial Distribution vs. Normal Approximation")
    plt.show()

from typing import Tuple
import math

def normal_approximation_to_binomial(n: int, p: float) -> Tuple[float, float]:
    """Returns mu and sigma corresponding to a Binomial(n, p)"""
    mu = p * n
    sigma = math.sqrt(p * (1 - p) * n)
    return mu, sigma

from scratch.probability import normal_cdf

# The normal cdf _is_ the probability the variable is below a threshold
normal_probability_below = normal_cdf

# It's above the threshold if it's not below the threshold
def normal_probability_above(lo: float,
                             mu: float = 0,
                             sigma: float = 1) -> float:
    """The probability that an N(mu, sigma) is greater than lo."""
    return 1 - normal_cdf(lo, mu, sigma)

# It's between if it's less than hi, but not less than lo
def normal_probability_between(lo: float,
                               hi: float,
                               mu: float = 0,
                               sigma: float = 1) -> float:
    """The probability that an N(mu, sigma) is between lo and hi."""
    return normal_cdf(hi, mu, sigma) - normal_cdf(lo, mu, sigma)

# It's outside if it's not between
def normal_probability_outside(lo: float,
                               hi: float,
                               mu: float = 0,
                               sigma: float = 1) -> float:
    """The probability that an N(mu, sigma) is not between lo and hi."""
    return 1 - normal_probability_between(lo, hi, mu, sigma)

from scratch.probability import inverse_normal_cdf

def normal_upper_bound(probability: float,
                       mu: float = 0,
                       sigma: float = 1) -> float:
    """Returns the z for which P(Z <= z) = probability"""
    return inverse_normal_cdf(probability, mu, sigma)

def normal_lower_bound(probability: float,
                       mu: float = 0,
                       sigma: float = 1) -> float:
    """Returns the z for which P(Z >= z) = probability"""
    return inverse_normal_cdf(1 - probability, mu, sigma)

def normal_two_sided_bounds(probability: float,
                            mu: float = 0,
                            sigma: float = 1) -> Tuple[float, float]:
    """
    Returns the symmetric (about the mean) bounds
    that contain the specified probability
    """
    tail_probability = (1 - probability) / 2

    # upper bound should have tail_probability above it
    upper_bound = normal_lower_bound(tail_probability, mu, sigma)

    # lower bound should have tail_probability below it
    lower_bound = normal_upper_bound(tail_probability, mu, sigma)

    return lower_bound, upper_bound

	# 95% bounds based on assumption p is 0.5
	#(469, 531)
	lo, hi = normal_two_sided_bounds(0.95, mu_0, sigma_0)
	
	# actual mu and sigma based on p = 0.55
	mu_1, sigma_1 = normal_approximation_to_binomial(1000, 0.55)
	
	# a type 2 error means we fail to reject the null hypothesis,
	# which will happen when X is still in our original interval
	type_2_probability = normal_probability_between(lo, hi, mu_1, sigma_1)
	power = 1 - type_2_probability      # 0.887
	```
	
 - 
	- We reject H_0 when we see 


### 7.3 Approach 2: p-Values
- We compute the probability, assuming $H_0$ is true, that we see a value as extreme as the one we actually observed.

```python
def two_sided_p_value(x: float, mu: float = 0, sigma: float = 1) -> float:
    """
    How likely are we to see a value at least as extreme as x (in either
    direction) if our values are from an N(mu, sigma)?
    """
    if x >= mu:
        # x is greater than the mean, so the tail is everything greater than x
        return 2 * normal_probability_above(x, mu, sigma)
    else:
        # x is less than the mean, so the tail is everything less than x
        return 2 * normal_probability_below(x, mu, sigma)

	def estimated_parameters(N: int, n: int) -> Tuple[float, float]:
	    p = n / N
	    sigma = math.sqrt(p * (1 - p) / N)
	    return p, sigma
	```


- Calculate the difference of $\mu , \sigma$
	- The 2 normal distribution A and B are independent
		- $p_{diff} = p_a - p_b$
		- $\sigma_{diff} = \sqrt{\sigma_a^2 + \sigma_b^2}$
- Our Null hypothesis is $p_A$ and $p_B$ are the same
```python
	# returns the z value
	def a_b_test_statistic(N_A: int, n_A: int, N_B: int, n_B: int) -> float:
	    p_A, sigma_A = estimated_parameters(N_A, n_A)
	    p_B, sigma_B = estimated_parameters(N_B, n_B)
	    return (p_B - p_A) / math.sqrt(sigma_A ** 2 + sigma_B ** 2)

z = a_b_test_statistic(1000, 200, 1000, 180)    # -1.14
two_sided_p_value(z)                            # 0.254

z = a_b_test_statistic(1000, 200, 1000, 150)    # -2.94
two_sided_p_value(z)                            # 0.003

from typing import Callable

# returns gradient
def gradient_scalar(f: Callable[[float], float],
                        x: float,
                        h: float) -> float:
    return (f(x + h) - f(x)) / h

# Funtion we are trying to calculate the derivative
def square(x: float) -> float:
    return x * x

# Test
xs = range(-10, 11)
estimates = [gradient_scalar(square, x, h=0.001) for x in xs]

# plot to show they're basically the same --> shows a line with a slope of 2
import matplotlib.pyplot as plt
plt.title("Actual Derivatives vs. Estimates")
plt.plot(xs, estimates, 'b+', label='Estimate')   # blue +
plt.legend(loc=9)
plt.show()

def gradient_vector(f: Callable[[Vector], float],
                                v: Vector,
                                i: int,
                                h: float) -> float:
    """Returns the i-th partial difference quotient of f at v"""
    w = [v_j + (h if j == i else 0)    # add h to just the ith element of v
         for j, v_j in enumerate(v)]

    return (f(w) - f(v)) / h


def estimate_gradient(f: Callable[[Vector], float],
                      v: Vector,
                      h: float = 0.0001):
    return [gradient_vector(f, v, i, h)
            for i in range(len(v))]

import random
from scratch.linear_algebra import distance, add, scalar_multiply

def gradient_step(v: Vector, gradient: Vector, step_size: float) -> Vector:
    """Moves `step_size` in the `gradient` direction from `v`"""
    assert len(v) == len(gradient)
    step = scalar_multiply(step_size, gradient)
    return add(v, step)

# pick a random starting point
v = [random.uniform(-10, 10) for i in range(3)]

# let f be the function we are trying to optimize
for epoch in range(1000):
	# compute the gradient at v
    grad = estimate_gradient(lambda x: x * x , v, step_size=0.001)    
    v = gradient_step(v, grad, -0.01)    # take a negative gradient step
    print(epoch, v)

# Method clculates the error between prediction and actual
def linear_gradient(x: float, y: float, theta: Vector) -> Vector:
    slope, intercept = theta.            # theta is the parameter vector we are learning
    predicted = slope * x + intercept    # The prediction of the model.
    error = (predicted - y)              # error is (predicted - actual).
    squared_error = error ** 2           # We'll minimize squared error
    grad = [2 * error * x, 2 * error]    # using its gradient.
    return grad

from scratch.linear_algebra import vector_mean

# Start with random values for slope and intercept
theta = [random.uniform(-1, 1), random.uniform(-1, 1)]

learning_rate = 0.001

for epoch in range(5000):
    # Compute the mean of the gradients
    grad = vector_mean([linear_gradient(x, y, theta) for x, y in inputs])
    # Take a step in that direction
    theta = gradient_step(theta, grad, -learning_rate)
    print(epoch, theta)

	# most_common_words.py
	import sys
	from collections import Counter
	
	# pass in number of words as first argument
	try:
	    num_words = int(sys.argv[1])
	except:
	    print("usage: most_common_words.py num_words")
	    sys.exit(1)   # nonzero exit code indicates error
	
	counter = Counter(word.lower()                      # lowercase words
	                  for line in sys.stdin
	                  for word in line.strip().split()  # split on spaces
	                  if word)                          # skip empty 'words'
	
	for word, count in counter.most_common(num_words):
	    sys.stdout.write(str(count))
	    sys.stdout.write("\t")
	    sys.stdout.write(word)
	    sys.stdout.write("\n")
	```


### 9.2 Reading Files
- Basic: Read all the domains of a file
	```python
	def get_domain(email_address: str) -> str:
	    """Split on '@' and return the last piece"""
	    return email_address.lower().split("@")[-1]
	
	# a couple of tests
	assert get_domain('joelgrus@gmail.com') == 'gmail.com'
	assert get_domain('joel@m.datasciencester.com') == 'm.datasciencester.com'
	
	from collections import Counter
	
	with open('email_addresses.txt', 'r') as f:
	    domain_counts = Counter(get_domain(line.strip())
	                            for line in f
	                            if "@" in line)
	```


### 9.3 Scraping the Web
- Tools
	- Use Beautiful Soup Library to build a tree out of the web page
	- Use Request library to make HTTP request
	- Parser:  html5lib
	- python -m pip install beautifulsoup4 requests html5lib

- Ex: Scrape house of representative _[web](https://www.house.gov/representatives)_ who mentioned data in their press link
	_HTML: view source_
	```html
	<td>
	  <a href="https://jayapal.house.gov">Jayapal, Pramila</a>
	</td>
	```

	_Code 1: Find the Valid URLS of representative_
	```python
	from bs4 import BeautifulSoup
	import requests
	
	url = "https://www.house.gov/representatives"
	text = requests.get(url).text
	soup = BeautifulSoup(text, "html5lib")

	# Must start with http:// or https://
	# Must end with .house.gov or .house.gov/
	regex = r"^https?://.*\.house\.gov/?$"

	# Let's write some tests!
	assert re.match(regex, "http://joel.house.gov")
	assert re.match(regex, "https://joel.house.gov")

	all_urls = [a['href']
	            for a in soup('a')
	            if a.has_attr('href')]
	
	# And now apply
	good_urls = [url for url in all_urls if re.match(regex, url)]

	print(len(set(all_urls)))  # 965 for me, way too many

	from typing import Dict, Set
	
	press_releases: Dict[str, Set[str]] = {}
	
	for house_url in good_urls:
	    html = requests.get(house_url).text
	    soup = BeautifulSoup(html, 'html5lib')
	    pr_links = {a['href'] for a in soup('a') if 'press releases'
	                                             in a.text.lower()}
	    print(f"{house_url}: {pr_links}")
	    press_releases[house_url] = pr_links

for house_url, pr_links in press_releases.items():
    for pr_link in pr_links:
        url = f"{house_url}/{pr_link}"
        text = requests.get(url).text

        if paragraph_mentions(text, 'data'):
            print(f"{house_url}")
            break  # done with this house_url

# Search for tweets containing the phrase "data science"
for status in twitter.search(q='"data science"')["statuses"]:
    user = status["user"]["screen_name"]
    text = status["text"]
    print(f"{user}: {text}\n")

from twython import TwythonStreamer

# Appending data to a global variable is pretty poor form
# but it makes the example much simpler
tweets = []

class MyStreamer(TwythonStreamer):
    def on_success(self, data):
        """
        What do we do when Twitter sends us data?
        Here data will be a Python dict representing a tweet.
        """
        # We only want to collect English-language tweets
        if data.get('lang') == 'en':
            tweets.append(data)
            print(f"received tweet #{len(tweets)}")

        # Stop when we've collected enough
        if len(tweets) >= 100:
            self.disconnect()

    def on_error(self, status_code, data):
        print(status_code, data)
        self.disconnect()

stream = MyStreamer(CONSUMER_KEY, CONSUMER_SECRET,
                    ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

# starts consuming public statuses that contain the keyword 'data'
stream.statuses.filter(track='data')

# if instead we wanted to start consuming a sample of *all* public statuses
# stream.statuses.sample()


top_hashtags = Counter(hashtag['text'].lower()
                       for tweet in tweets
                       for hashtag in tweet["entities"]["hashtags"])

print(top_hashtags.most_common(5))

	from typing import List, Dict
	from collections import Counter
	import math
	
	import matplotlib.pyplot as plt
	
	def bucketize(point: float, bucket_size: float) -> float:
	    """Floor the point to the next lower multiple of bucket_size"""
	    return bucket_size * math.floor(point / bucket_size)
	
	def make_histogram(points: List[float], bucket_size: float) -> Dict[float, int]:
	    """Buckets the points and counts how many in each bucket"""
	    return Counter(bucketize(point, bucket_size) for point in points)
	
	def plot_histogram(points: List[float], bucket_size: float, title: str = ""):
	    histogram = make_histogram(points, bucket_size)
	    plt.bar(histogram.keys(), histogram.values(), width=bucket_size)
	    plt.title(title)


# normal distribution with mean 0, standard deviation 57
import random
# uniform between -100 and 100
uniform = [200 * random.random() - 100 for _ in range(10000)]
plot_histogram(uniform, 10, "Uniform Histogram")

# Define 2 axis input
def random_normal() -> float:
    """Returns a random draw from a standard normal distribution"""
    return inverse_normal_cdf(random.random())

xs = [random_normal() for _ in range(1000)]
ys1 = [ x + random_normal() / 2 for x in xs]
ys2 = [-x + random_normal() / 2 for x in xs]

plt.scatter(xs, ys1, marker='.', color='black', label='ys1')
plt.scatter(xs, ys2, marker='.', color='gray',  label='ys2')
plt.xlabel('xs')
plt.ylabel('ys')
plt.legend(loc=9)
plt.title("Very Different Joint Distributions")
plt.show()

from scratch.linear_algebra import Matrix, Vector, make_matrix

def correlation_matrix(data: List[Vector]) -> Matrix:
    """
    Returns the len(data) x len(data) matrix whose (i, j)-th entry
    is the correlation between data[i] and data[j]
    """
    def correlation_ij(i: int, j: int) -> float:
        return correlation(data[i], data[j])

    return make_matrix(len(data), len(data), correlation_ij)


# corr_data is a list of four 100-d vectors
num_vectors = len(corr_data)
fig, ax = plt.subplots(num_vectors, num_vectors)

for i in range(num_vectors):
    for j in range(num_vectors):

        # Scatter column_j on the x-axis vs. column_i on the y-axis
        if i != j: ax[i][j].scatter(corr_data[j], corr_data[i])

        # unless i == j, in which case show the series name
        else: ax[i][j].annotate("series " + str(i), (0.5, 0.5),
                                xycoords='axes fraction',
                                ha="center", va="center")

        # Then hide axis labels except left and bottom charts
        if i < num_vectors - 1: ax[i][j].xaxis.set_visible(False)
        if j > 0: ax[i][j].yaxis.set_visible(False)

# Fix the bottom-right and top-left axis labels, which are wrong because
# their charts only have text in them
ax[-1][-1].set_xlim(ax[0][-1].get_xlim())
ax[0][0].set_ylim(ax[0][1].get_ylim())

plt.show()