from bs4 import BeautifulSoup
import requests
import pandas as pd
import matplotlib.pyplot as plt
import squarify
# Fetch Wikipedia page that we want to scrap
url = 'https://en.wikipedia.org/wiki/List_of_largest_companies_in_the_United_States_by_revenue'
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html')
print(soup)
# Print the entire page
<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available" dir="ltr" lang="en"> <head> </head> <body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-List_of_largest_companies_in_the_United_States_by_revenue rootpage-List_of_largest_companies_in_the_United_States_by_revenue skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav aria-label="Site" class="vector-main-menu-landmark"> <div class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" id="vector-main-menu-dropdown"> <input aria-haspopup="true" aria-label="Main menu" class="vector-dropdown-checkbox" data-event-name="ui.dropdown-vector-main-menu-dropdown" id="vector-main-menu-dropdown-checkbox" role="button" type="checkbox"/> <label aria-hidden="true" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" for="vector-main-menu-dropdown-checkbox" id="vector-main-menu-dropdown-label"><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div class="vector-unpinned-container" id="vector-main-menu-unpinned-container"> <div class="vector-main-menu vector-pinnable-element" id="vector-main-menu"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container"> <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> ........... </body> </html>
soup.find_all('table')[0]
# There are multiple tables on the page. (0, 1, and 2)
# Use indexing to find the table that we are interested in, in this case the first (0) table
<table class="wikitable sortable"> <caption> </caption> <tbody><tr> <th>Rank </th> <th>Name </th> <th>Industry </th> <th>Revenue <br/>(USD millions) </th> <th>Revenue growth </th> <th>Employees </th> <th>Headquarters </th></tr> <tr> <td>1 </td> <td><a href="/wiki/Walmart" title="Walmart">Walmart</a> </td> <td><a href="/wiki/Retail" title="Retail">Retail</a> </td> <td style="text-align:center;">648,125 </td> <td style="text-align:center;"><span typeof="mw:File"><span title="Increase"><img alt="Increase" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/11px-Increase2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/17px-Increase2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/22px-Increase2.svg.png 2x" width="11"/></span></span> <span data-sort-value="7000300000000000000♠" style="display:none"></span> 6.0% </td> <td style="text-align:center;">2,100,000 </td> <td><a href="/wiki/Bentonville,_Arkansas" title="Bentonville, Arkansas">Bentonville, Arkansas</a> </td></tr> <tr> <td>2 </td> <td><a href="/wiki/Amazon_(company)" title="Amazon (company)">Amazon</a> </td> <td>Retail and <a href="/wiki/Cloud_computing" title="Cloud computing">cloud computing</a> </td> <td style="text-align:center;">574,785 </td> <td style="text-align:center;"><span typeof="mw:File"><span title="Increase"><img alt="Increase" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/11px-Increase2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/17px-Increase2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/22px-Increase2.svg.png 2x" width="11"/></span></span> <span data-sort-value="7000300000000000000♠" style="display:none"></span> 11.9% </td> <td style="text-align:center;">1,525,000 </td> <td><a href="/wiki/Seattle" title="Seattle">Seattle, Washington</a> </td></tr> <tr> <td>3 </td> <td><a class="mw-redirect" href="/wiki/Apple_Inc" title="Apple Inc">Apple</a> </td> <td><a href="/wiki/Electronics_industry" title="Electronics industry">Electronics industry</a> </td> <td style="text-align:center;">383,482 </td> <td style="text-align:center;"><span typeof="mw:File"><span title="Decrease"><img alt="Decrease" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/11px-Decrease2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/17px-Decrease2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/22px-Decrease2.svg.png 2x" width="11"/></span></span> <span data-sort-value="2999450000000000000♠" style="display:none"></span> -2.8% </td> <td style="text-align:center;">161,000 </td> <td><a href="/wiki/Cupertino,_California" title="Cupertino, California">Cupertino, California</a> </td></tr> <tr> ........... </td></tr></tbody></table>
soup.find('table', class_ = 'wikitable sortable')
# Alternatively, we can find the table by using its class name
<table class="wikitable sortable"> <caption> </caption> <tbody><tr> <th>Rank </th> <th>Name </th> <th>Industry </th> <th>Revenue <br/>(USD millions) </th> <th>Revenue growth </th> <th>Employees </th> <th>Headquarters </th></tr> <tr> <td>1 </td> <td><a href="/wiki/Walmart" title="Walmart">Walmart</a> </td> <td><a href="/wiki/Retail" title="Retail">Retail</a> </td> <td style="text-align:center;">648,125 </td> <td style="text-align:center;"><span typeof="mw:File"><span title="Increase"><img alt="Increase" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/11px-Increase2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/17px-Increase2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/22px-Increase2.svg.png 2x" width="11"/></span></span> <span data-sort-value="7000300000000000000♠" style="display:none"></span> 6.0% </td> <td style="text-align:center;">2,100,000 </td> <td><a href="/wiki/Bentonville,_Arkansas" title="Bentonville, Arkansas">Bentonville, Arkansas</a> </td></tr> <tr> <td>2 </td> <td><a href="/wiki/Amazon_(company)" title="Amazon (company)">Amazon</a> </td> <td>Retail and <a href="/wiki/Cloud_computing" title="Cloud computing">cloud computing</a> </td> <td style="text-align:center;">574,785 </td> <td style="text-align:center;"><span typeof="mw:File"><span title="Increase"><img alt="Increase" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/11px-Increase2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/17px-Increase2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/22px-Increase2.svg.png 2x" width="11"/></span></span> <span data-sort-value="7000300000000000000♠" style="display:none"></span> 11.9% </td> <td style="text-align:center;">1,525,000 </td> <td><a href="/wiki/Seattle" title="Seattle">Seattle, Washington</a> </td></tr> <tr> <td>3 </td> <td><a class="mw-redirect" href="/wiki/Apple_Inc" title="Apple Inc">Apple</a> </td> <td><a href="/wiki/Electronics_industry" title="Electronics industry">Electronics industry</a> </td> <td style="text-align:center;">383,482 </td> <td style="text-align:center;"><span typeof="mw:File"><span title="Decrease"><img alt="Decrease" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/11px-Decrease2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/17px-Decrease2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/22px-Decrease2.svg.png 2x" width="11"/></span></span> <span data-sort-value="2999450000000000000♠" style="display:none"></span> -2.8% </td> <td style="text-align:center;">161,000 </td> <td><a href="/wiki/Cupertino,_California" title="Cupertino, California">Cupertino, California</a> </td></tr> <tr> ........... </td></tr></tbody></table>
# Find the first table on the page
table = soup.find_all('table')[0]
# Extract column headers
# Find the th tag in the table & loop through the th tags
titles = table.find_all('th')
table_titles = [title.text.strip() for title in titles]
print(table_titles)
# This will print all the table headers
['Rank', 'Name', 'Industry', 'Revenue (USD millions)', 'Revenue growth', 'Employees', 'Headquarters']
# Create pandas dataframe from the headers
df = pd.DataFrame(columns=table_titles)
df
| Rank | Name | Industry | Revenue (USD millions) | Revenue growth | Employees | Headquarters |
|---|
column_data = table.find_all('tr')
# Extract rows from the table
column_data = table.find_all('tr')
for row in column_data[1:]:
row_data = row.find_all('td')
individual_row_data = [data.text.strip() for data in row_data]
length = len(df)
df.loc[length] = individual_row_data
df
| Rank | Name | Industry | Revenue (USD millions) | Revenue growth | Employees | Headquarters | |
|---|---|---|---|---|---|---|---|
| 0 | 1 | Walmart | Retail | 648,125 | 6.0% | 2,100,000 | Bentonville, Arkansas |
| 1 | 2 | Amazon | Retail and cloud computing | 574,785 | 11.9% | 1,525,000 | Seattle, Washington |
| 2 | 3 | Apple | Electronics industry | 383,482 | -2.8% | 161,000 | Cupertino, California |
| 3 | 4 | UnitedHealth Group | Healthcare | 371,622 | 14.6% | 440,000 | Minnetonka, Minnesota |
| 4 | 5 | Berkshire Hathaway | Conglomerate | 364,482 | 20.7% | 396,500 | Omaha, Nebraska |
| ... | ... | ... | ... | ... | ... | ... | ... |
| 95 | 96 | TIAA | Financials | 45,735 | 11.8% | 16,023 | New York City, New York |
| 96 | 97 | CHS | Agriculture cooperative | 45,590 | -4.6% | 10,609 | Inver Grove Heights, Minnesota |
| 97 | 98 | Bristol-Myers Squibb | Pharmaceutical industry | 45,006 | -2.5% | 34,100 | New York City, New York |
| 98 | 99 | Dow Chemical Company | Chemical industry | 44,622 | -21.6% | 35,900 | Midland, Michigan |
| 99 | 100 | Best Buy | Retail | 43,452 | -6.1% | 85,000 | Richfield, Minnesota |
100 rows × 7 columns
# Clean and convert the "Revenue (USD millions)" column from string to numerical values
df['Revenue (USD millions)'] = df['Revenue (USD millions)'].replace('[\$,]', '', regex=True).astype(float)
# List the unique values in the "Industry" column, ordered by their frequency
industry_counts = df['Industry'].value_counts()
# Display the industries ordered by their frequency
print(industry_counts)
# We will use the top 10 industries and remaining will be 'others'
Industry Financials 13 Retail 10 Petroleum industry 9 Pharmaceutical industry 6 Technology 6 Healthcare 6 Insurance 5 Telecommunications 3 Airline 3 Technology and cloud computing 3 Conglomerate 3 Food processing 2 Beverage 2 Automotive industry 2 Financial 2 Aerospace and defense 2 Transportation 2 Health insurance 2 Agriculture manufacturing 1 Agriculture cooperative 1 Infotech 1 Telecom hardware manufacturing 1 Apparel 1 Energy trading 1 Machinery 1 Food industry 1 Food service 1 Consumer products manufacturing 1 Media 1 Retail and cloud computing 1 Automotive and energy 1 Conglomerate and telecommunications 1 Financial services 1 Pharmacy wholesale 1 Health 1 Electronics industry 1 Chemical industry 1 Name: count, dtype: int64
# Next, we will create a Treemap using the ompany name and revenue values
# Define the top 10 industries and a color mapping for them
industry_color_map = {
'Financials': '#1f77b4', # Blue
'Retail': '#ff7f0e', # Orange
'Petroleum industry': '#2ca02c', # Green
'Pharmaceutical industry': '#d62728', # Red
'Technology': '#9467bd', # Purple
'Healthcare': '#8c564b', # Brown
'Insurance': '#e377c2', # Pink
'Telecommunications': '#7f7f7f', # Gray
'Airline': '#bcbd22', # Yellow-green
'Technology and cloud computing': '#17becf' # Cyan
}
# If an industry is not in the color map, categorize it as 'Others'
df['Industry'] = df['Industry'].apply(lambda x: x if x in industry_color_map else 'Others')
# Update the color map to include 'Others'
industry_color_map['Others'] = '#aec7e8' # Light Blue for 'Others'
# Clean and filter the dataframe
df_cleaned = df.dropna(subset=['Revenue (USD millions)', 'Industry'])
df_cleaned = df_cleaned[df_cleaned['Revenue (USD millions)'] > 0]
# Normalize revenue values for better sizing in the treemap
normed_revenue = df_cleaned['Revenue (USD millions)']
# Map colors to industries
colors = df_cleaned['Industry'].map(industry_color_map)
# Create Treemap using squarify
plt.figure(figsize=(12, 8))
squarify.plot(sizes=normed_revenue,
label=df_cleaned['Name'],
color=colors,
alpha=0.8)
# Add title and clean up axis
plt.title('Top 100 Companies by Revenue')
plt.axis('off')
plt.show()