In [62]:
from bs4 import BeautifulSoup
import requests
import pandas as pd
import matplotlib.pyplot as plt
import squarify
In [63]:
# Fetch Wikipedia page that we want to scrap
url = 'https://en.wikipedia.org/wiki/List_of_largest_companies_in_the_United_States_by_revenue'
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html')
In [4]:
print(soup)
# Print the entire page
<!DOCTYPE html>
<html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-enabled skin-theme-clientpref-day vector-toc-available" dir="ltr" lang="en">
<head>

</head>
<body class="skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-List_of_largest_companies_in_the_United_States_by_revenue rootpage-List_of_largest_companies_in_the_United_States_by_revenue skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a>
<div class="vector-header-container">
<header class="vector-header mw-header">
<div class="vector-header-start">
<nav aria-label="Site" class="vector-main-menu-landmark">
<div class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" id="vector-main-menu-dropdown">
<input aria-haspopup="true" aria-label="Main menu" class="vector-dropdown-checkbox" data-event-name="ui.dropdown-vector-main-menu-dropdown" id="vector-main-menu-dropdown-checkbox" role="button" type="checkbox"/>
<label aria-hidden="true" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only" for="vector-main-menu-dropdown-checkbox" id="vector-main-menu-dropdown-label"><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span>
<span class="vector-dropdown-label-text">Main menu</span>
</label>
<div class="vector-dropdown-content">
<div class="vector-unpinned-container" id="vector-main-menu-unpinned-container">
<div class="vector-main-menu vector-pinnable-element" id="vector-main-menu">
<div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container">
<div class="vector-pinnable-header-label">Main menu</div>
<button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button>
<button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button>
</div>



...........



</body>
</html>
In [45]:
soup.find_all('table')[0]
# There are multiple tables on the page. (0, 1, and 2)
# Use indexing to find the table that we are interested in, in this case the first (0) table
Out[45]:
<table class="wikitable sortable">
<caption>
</caption>
<tbody><tr>
<th>Rank
</th>
<th>Name
</th>
<th>Industry
</th>
<th>Revenue <br/>(USD millions)
</th>
<th>Revenue growth
</th>
<th>Employees
</th>
<th>Headquarters
</th></tr>
<tr>
<td>1
</td>
<td><a href="/wiki/Walmart" title="Walmart">Walmart</a>
</td>
<td><a href="/wiki/Retail" title="Retail">Retail</a>
</td>
<td style="text-align:center;">648,125
</td>
<td style="text-align:center;"><span typeof="mw:File"><span title="Increase"><img alt="Increase" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/11px-Increase2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/17px-Increase2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/22px-Increase2.svg.png 2x" width="11"/></span></span> <span data-sort-value="7000300000000000000♠" style="display:none"></span> 6.0%
</td>
<td style="text-align:center;">2,100,000
</td>
<td><a href="/wiki/Bentonville,_Arkansas" title="Bentonville, Arkansas">Bentonville, Arkansas</a>
</td></tr>
<tr>
<td>2
</td>
<td><a href="/wiki/Amazon_(company)" title="Amazon (company)">Amazon</a>
</td>
<td>Retail and <a href="/wiki/Cloud_computing" title="Cloud computing">cloud computing</a>
</td>
<td style="text-align:center;">574,785
</td>
<td style="text-align:center;"><span typeof="mw:File"><span title="Increase"><img alt="Increase" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/11px-Increase2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/17px-Increase2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/22px-Increase2.svg.png 2x" width="11"/></span></span> <span data-sort-value="7000300000000000000♠" style="display:none"></span> 11.9%
</td>
<td style="text-align:center;">1,525,000
</td>
<td><a href="/wiki/Seattle" title="Seattle">Seattle, Washington</a>
</td></tr>
<tr>
<td>3
</td>
<td><a class="mw-redirect" href="/wiki/Apple_Inc" title="Apple Inc">Apple</a>
</td>
<td><a href="/wiki/Electronics_industry" title="Electronics industry">Electronics industry</a>
</td>
<td style="text-align:center;">383,482
</td>
<td style="text-align:center;"><span typeof="mw:File"><span title="Decrease"><img alt="Decrease" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/11px-Decrease2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/17px-Decrease2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/22px-Decrease2.svg.png 2x" width="11"/></span></span> <span data-sort-value="2999450000000000000♠" style="display:none"></span> -2.8%
</td>
<td style="text-align:center;">161,000
</td>
<td><a href="/wiki/Cupertino,_California" title="Cupertino, California">Cupertino, California</a>
</td></tr>
<tr>



...........



</td></tr></tbody></table>
In [46]:
soup.find('table', class_ = 'wikitable sortable')
# Alternatively, we can find the table by using its class name
Out[46]:
<table class="wikitable sortable">
<caption>
</caption>
<tbody><tr>
<th>Rank
</th>
<th>Name
</th>
<th>Industry
</th>
<th>Revenue <br/>(USD millions)
</th>
<th>Revenue growth
</th>
<th>Employees
</th>
<th>Headquarters
</th></tr>
<tr>
<td>1
</td>
<td><a href="/wiki/Walmart" title="Walmart">Walmart</a>
</td>
<td><a href="/wiki/Retail" title="Retail">Retail</a>
</td>
<td style="text-align:center;">648,125
</td>
<td style="text-align:center;"><span typeof="mw:File"><span title="Increase"><img alt="Increase" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/11px-Increase2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/17px-Increase2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/22px-Increase2.svg.png 2x" width="11"/></span></span> <span data-sort-value="7000300000000000000♠" style="display:none"></span> 6.0%
</td>
<td style="text-align:center;">2,100,000
</td>
<td><a href="/wiki/Bentonville,_Arkansas" title="Bentonville, Arkansas">Bentonville, Arkansas</a>
</td></tr>
<tr>
<td>2
</td>
<td><a href="/wiki/Amazon_(company)" title="Amazon (company)">Amazon</a>
</td>
<td>Retail and <a href="/wiki/Cloud_computing" title="Cloud computing">cloud computing</a>
</td>
<td style="text-align:center;">574,785
</td>
<td style="text-align:center;"><span typeof="mw:File"><span title="Increase"><img alt="Increase" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/11px-Increase2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/17px-Increase2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/b/b0/Increase2.svg/22px-Increase2.svg.png 2x" width="11"/></span></span> <span data-sort-value="7000300000000000000♠" style="display:none"></span> 11.9%
</td>
<td style="text-align:center;">1,525,000
</td>
<td><a href="/wiki/Seattle" title="Seattle">Seattle, Washington</a>
</td></tr>
<tr>
<td>3
</td>
<td><a class="mw-redirect" href="/wiki/Apple_Inc" title="Apple Inc">Apple</a>
</td>
<td><a href="/wiki/Electronics_industry" title="Electronics industry">Electronics industry</a>
</td>
<td style="text-align:center;">383,482
</td>
<td style="text-align:center;"><span typeof="mw:File"><span title="Decrease"><img alt="Decrease" class="mw-file-element" data-file-height="300" data-file-width="300" decoding="async" height="11" src="//upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/11px-Decrease2.svg.png" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/17px-Decrease2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/e/ed/Decrease2.svg/22px-Decrease2.svg.png 2x" width="11"/></span></span> <span data-sort-value="2999450000000000000♠" style="display:none"></span> -2.8%
</td>
<td style="text-align:center;">161,000
</td>
<td><a href="/wiki/Cupertino,_California" title="Cupertino, California">Cupertino, California</a>
</td></tr>
<tr>



...........



</td></tr></tbody></table>
In [64]:
# Find the first table on the page
table = soup.find_all('table')[0]

# Extract column headers
# Find the th tag in the table & loop through the th tags
titles = table.find_all('th')
table_titles = [title.text.strip() for title in titles]

print(table_titles)
# This will print all the table headers
['Rank', 'Name', 'Industry', 'Revenue (USD millions)', 'Revenue growth', 'Employees', 'Headquarters']
In [65]:
# Create pandas dataframe from the headers
df = pd.DataFrame(columns=table_titles)

df
Out[65]:
Rank Name Industry Revenue (USD millions) Revenue growth Employees Headquarters
In [54]:
column_data = table.find_all('tr')
In [66]:
# Extract rows from the table
column_data = table.find_all('tr')
for row in column_data[1:]:
    row_data = row.find_all('td')
    individual_row_data = [data.text.strip() for data in row_data]

    length = len(df)
    df.loc[length] = individual_row_data
In [67]:
df
Out[67]:
Rank Name Industry Revenue (USD millions) Revenue growth Employees Headquarters
0 1 Walmart Retail 648,125 6.0% 2,100,000 Bentonville, Arkansas
1 2 Amazon Retail and cloud computing 574,785 11.9% 1,525,000 Seattle, Washington
2 3 Apple Electronics industry 383,482 -2.8% 161,000 Cupertino, California
3 4 UnitedHealth Group Healthcare 371,622 14.6% 440,000 Minnetonka, Minnesota
4 5 Berkshire Hathaway Conglomerate 364,482 20.7% 396,500 Omaha, Nebraska
... ... ... ... ... ... ... ...
95 96 TIAA Financials 45,735 11.8% 16,023 New York City, New York
96 97 CHS Agriculture cooperative 45,590 -4.6% 10,609 Inver Grove Heights, Minnesota
97 98 Bristol-Myers Squibb Pharmaceutical industry 45,006 -2.5% 34,100 New York City, New York
98 99 Dow Chemical Company Chemical industry 44,622 -21.6% 35,900 Midland, Michigan
99 100 Best Buy Retail 43,452 -6.1% 85,000 Richfield, Minnesota

100 rows × 7 columns

In [69]:
# Clean and convert the "Revenue (USD millions)" column from string to numerical values
df['Revenue (USD millions)'] = df['Revenue (USD millions)'].replace('[\$,]', '', regex=True).astype(float)
In [74]:
# List the unique values in the "Industry" column, ordered by their frequency
industry_counts = df['Industry'].value_counts()

# Display the industries ordered by their frequency
print(industry_counts)

# We will use the top 10 industries and remaining will be 'others'
Industry
Financials                             13
Retail                                 10
Petroleum industry                      9
Pharmaceutical industry                 6
Technology                              6
Healthcare                              6
Insurance                               5
Telecommunications                      3
Airline                                 3
Technology and cloud computing          3
Conglomerate                            3
Food processing                         2
Beverage                                2
Automotive industry                     2
Financial                               2
Aerospace and defense                   2
Transportation                          2
Health insurance                        2
Agriculture manufacturing               1
Agriculture cooperative                 1
Infotech                                1
Telecom hardware manufacturing          1
Apparel                                 1
Energy trading                          1
Machinery                               1
Food industry                           1
Food service                            1
Consumer products manufacturing         1
Media                                   1
Retail and cloud computing              1
Automotive and energy                   1
Conglomerate and telecommunications     1
Financial services                      1
Pharmacy wholesale                      1
Health                                  1
Electronics industry                    1
Chemical industry                       1
Name: count, dtype: int64
In [77]:
# Next, we will create a Treemap using the ompany name and revenue values

# Define the top 10 industries and a color mapping for them
industry_color_map = {
    'Financials': '#1f77b4',           # Blue
    'Retail': '#ff7f0e',               # Orange
    'Petroleum industry': '#2ca02c',   # Green
    'Pharmaceutical industry': '#d62728', # Red
    'Technology': '#9467bd',           # Purple
    'Healthcare': '#8c564b',           # Brown
    'Insurance': '#e377c2',            # Pink
    'Telecommunications': '#7f7f7f',   # Gray
    'Airline': '#bcbd22',              # Yellow-green
    'Technology and cloud computing': '#17becf'  # Cyan
}

# If an industry is not in the color map, categorize it as 'Others'
df['Industry'] = df['Industry'].apply(lambda x: x if x in industry_color_map else 'Others')

# Update the color map to include 'Others'
industry_color_map['Others'] = '#aec7e8'  # Light Blue for 'Others'

# Clean and filter the dataframe
df_cleaned = df.dropna(subset=['Revenue (USD millions)', 'Industry'])
df_cleaned = df_cleaned[df_cleaned['Revenue (USD millions)'] > 0]

# Normalize revenue values for better sizing in the treemap
normed_revenue = df_cleaned['Revenue (USD millions)']

# Map colors to industries
colors = df_cleaned['Industry'].map(industry_color_map)

# Create Treemap using squarify
plt.figure(figsize=(12, 8))
squarify.plot(sizes=normed_revenue,
              label=df_cleaned['Name'],
              color=colors,
              alpha=0.8)

# Add title and clean up axis
plt.title('Top 100 Companies by Revenue')
plt.axis('off')
plt.show()