In [1]:
# Import packages
import pandas as pd
from IPython.display import display
import numpy as np
import re
# Suppress all warnings
import warnings
warnings.filterwarnings('ignore')
# Ensure all columns are displayed
pd.set_option('display.max_columns', None) # Show all columns
pd.set_option('display.width', None) # Adjust display width to avoid wrapping
pd.set_option('display.max_colwidth', None) # Show full content of each column (if needed)
In [ ]:
In [2]:
# I am using my pre-made functions to check and analyze the dataset given.
In [3]:
def analyze_dataset(df, dataset_name):
"""
Function to analyze a given dataset.
Displays basic dataset information and a preview of the data.
Parameters:
df (DataFrame): The DataFrame to analyze.
Returns:
None
"""
# Get the total number of rows and columns
print(f"\033[1m\nDataset: {dataset_name} \033[0m")
row_count, column_count = df.shape
print(f"\033[1m\nTotal number of rows: {row_count} \033[0m")
print(f"\033[1mTotal number of columns: {column_count} \033[0m\n")
# Display the first 5 rows of the dataset
print("\nFirst 3 rows of the dataset:")
display(df.head(n=3))
# Display all column names
print("\n\nAll Columns in the Dataset:")
print(df.columns.tolist())
# Identify and list repeated (duplicate) column names
repeated_columns = df.columns[df.columns.duplicated()].tolist()
# Display the repeated columns
if repeated_columns:
print("\nRepeated Columns:")
print(repeated_columns)
else:
print("\nNo repeated columns found.")
In [ ]:
In [4]:
def quality_issue_inspection_part1(df, dataset_name):
"""
Function to do quality inspection on a given dataset.
Displays missing values, duplicate rows, invalid values, outliers,
trailing spaces, column entries with mixed format, non-numeric data
in numeric columns, and numeric data in non-numeric columns.
Parameters:
df (DataFrame): The DataFrame to analyze.
dataset_name (str): Name of the dataset.
Returns:
None
"""
# Bold formatting
bold_start = "\033[1m"
bold_end = "\033[0m"
print(f"\n\n{bold_start} ---------------------- Quality Issues Inspection Part 1 ---------------------- {bold_end}\n")
print(f"{bold_start}Dataset:{bold_end} {dataset_name} \n\n")
# 1. Missing Values
print("1. Missing Values:")
# 1.a: Analyze Missing Values in Columns
missing_values = df.isnull().sum() # Total missing values per column
if missing_values.sum() == 0:
print("Missing Values: None\n")
else:
# Calculate the percentage of missing values for each column
missing_percentage = (missing_values / len(df)) * 100
# Create a summary DataFrame
missing_summary = pd.DataFrame({'Missing Values': missing_values, 'Percentage': missing_percentage.round(2)})
print("Missing Values Summary (Columns):")
print(missing_summary, "\n")
# 1.b: Analyze Missing Values in Rows (without permanently adding columns)
# Temporary calculation of total and percentage of missing values per row
df_temp = df.copy() # Create a temporary copy of the dataframe
df_temp['Total_Missing_Values'] = df.isnull().sum(axis=1) # Total missing values per row
df_temp['Missing_Percentage'] = (df_temp['Total_Missing_Values'] / len(df.columns)) * 100 # Percentage missing per row
# Total rows and rows with missing values
total_rows = df_temp.shape[0] # Total number of rows in the dataset
total_rows_with_missing = df_temp[df_temp['Total_Missing_Values'] > 0].shape[0]
# Percentage of rows with missing values
percentage_rows_with_missing = (total_rows_with_missing / total_rows) * 100
# Threshold for identifying rows with high missing values
row_threshold = 30 # Rows with >30% missing values
# Classify rows based on missing percentages
rows_high_missing = df_temp[df_temp['Missing_Percentage'] > row_threshold] # Rows with >threshold% missing
rows_low_missing = df_temp[(df_temp['Missing_Percentage'] > 0) & (df_temp['Missing_Percentage'] <= row_threshold)] # Rows with 0%-threshold%
rows_no_missing = df_temp[df_temp['Missing_Percentage'] == 0] # Rows with no missing values
# Display summary results for rows
print("Missing Values Summary (Rows):")
print(f" Total rows with missing values: {total_rows_with_missing} "
f"({percentage_rows_with_missing:.2f}%)")
print(f" Rows with high missing values (> {row_threshold}%): {len(rows_high_missing)} "
f"({(len(rows_high_missing) / total_rows * 100):.2f}%)")
print(f" Rows with low missing values (0% - {row_threshold}%): {len(rows_low_missing)} "
f"({(len(rows_low_missing) / total_rows * 100):.2f}%)")
print(f" Rows with no missing values: {len(rows_no_missing)} "
f"({(len(rows_no_missing) / total_rows * 100):.2f}%)\n")
# 2. Duplicate Rows
duplicate_rows = df.duplicated().sum()
print(f"2. Duplicate Rows: Number of duplicate rows: {duplicate_rows}" if duplicate_rows > 0 else "2. Duplicate Rows: None\n")
# 3. Invalid Values (Negative or Out-of-Range Values)
invalid_found = False
for col in df.select_dtypes(include=['int64', 'float64']):
invalid_rows = df[df[col] < 0]
if not invalid_rows.empty:
invalid_found = True
print(f"3. Invalid values in '{col}':")
print(invalid_rows[[col]].head(), "\n")
if not invalid_found:
print("3. Invalid and/or Negative Values: None\n")
# 4. Outliers (Using IQR)
print("4. Outliers")
# Combine outlier detection and column listing
outliers_found = False
numeric_cols = df.select_dtypes(include=['int64', 'float64']).columns
columns_with_outliers = []
total_outliers = 0 # Counter to track the total number of outliers
total_rows = len(df) # Total number of rows in the dataset
for col in numeric_cols:
Q1 = df[col].quantile(0.25) # First quartile
Q3 = df[col].quantile(0.75) # Third quartile
IQR = Q3 - Q1 # Interquartile Range
lower_bound = Q1 - 1.5 * IQR # Lower bound for outliers
upper_bound = Q3 + 1.5 * IQR # Upper bound for outliers
# Find rows with outliers in the column
outliers = df[(df[col] < lower_bound) | (df[col] > upper_bound)]
if not outliers.empty:
outliers_found = True
columns_with_outliers.append(col)
print(f"Outliers detected in '{col}': {len(outliers)} rows")
total_outliers += len(outliers) # Add to the total count of outliers
if not outliers_found:
print("No outliers detected in any column.\n")
# Print the list of columns with outliers
if columns_with_outliers:
print("\nColumns with outliers:", columns_with_outliers)
else:
print("\nNo columns with outliers found.")
# Calculate and print the total percentage of outliers
if total_outliers > 0:
outlier_percentage = (total_outliers / (total_rows * len(numeric_cols))) * 100
print(f"\nTotal number of outliers across all columns: {total_outliers}")
print(f"Percentage of total outliers in the dataset: {outlier_percentage:.2f}%\n")
else:
print("\nNo outliers found in the dataset.\n")
# 5. Redundant Leading/Trailing Spaces
print("\n5. Redundant Leading/Trailing Spaces:")
spaces_found = False
for col in df.select_dtypes(include=['object']):
spaces = df[col].str.contains(r'^\s|\s$', na=False)
if spaces.any():
spaces_found = True
print(f" Leading/trailing spaces found in column '{col}'.")
if not spaces_found:
print(" None\n")
# 6. Column Entries with Mixed Formats
print("6. Column Entries with Mixed Formats:")
mixed_formats_found = False
for col in df.columns:
if df[col].dtype == 'object':
mixed_format = df[col].apply(lambda x: bool(re.search(r'\d{4}-\d{2}-\d{2}', str(x))) if pd.notnull(x) else False)
if mixed_format.any():
mixed_formats_found = True
print(f" Mixed formats detected in '{col}':")
print(df[col][mixed_format].head(), "\n")
if not mixed_formats_found:
print(" None\n")
# 7. Non-Numeric Data in Numeric Columns
print("7. Non-Numeric Data in Numeric Columns:")
non_numeric_data = []
numeric_cols = df.select_dtypes(include=['int64', 'float64']).columns
for col in numeric_cols:
# Identify non-numeric data
invalid_data = df[pd.to_numeric(df[col], errors='coerce').isnull()][col]
if not invalid_data.empty:
non_numeric_data.append({
'Column': col,
'Non-Numeric Data': invalid_data.unique().tolist(),
'Count': invalid_data.shape[0]
})
if non_numeric_data:
# Create and display a summary DataFrame
non_numeric_summary = pd.DataFrame(non_numeric_data)
print(non_numeric_summary.to_string(index=False))
else:
print(" None\n")
# 8. Numeric Data in Non-Numeric Columns
print("\n8. Numeric Data in Non-Numeric Columns:")
numeric_in_non_numeric_data = []
non_numeric_cols = df.select_dtypes(include=['object']).columns
for col in non_numeric_cols:
# Identify numeric data in non-numeric columns
numeric_data = df[col].apply(lambda x: pd.to_numeric(x, errors='coerce')).dropna()
if not numeric_data.empty:
numeric_in_non_numeric_data.append({
'Column': col,
'Numeric Data': numeric_data.unique().tolist(),
'Count': numeric_data.shape[0]
})
if numeric_in_non_numeric_data:
# Create and display a summary DataFrame
numeric_data_summary = pd.DataFrame(numeric_in_non_numeric_data)
print(numeric_data_summary.to_string(index=False))
else:
print(" None\n")
In [ ]:
In [5]:
def quality_issue_inspection_part2(df, dataset_name):
"""
Analyze a dataset to detect:
1. Special characters or formatting issues in object columns.
2. Inconsistent units in object columns with recommendations.
3. Unique categorical values in object columns.
4. Analyze a dataset for data types and classification.
Displays the results in a tabular format and saves them as CSV files.
Args:
df (pd.DataFrame): The dataset to analyze.
dataset_name (str): Name of the dataset.
Returns:
1. Summary DataFrame with data types and classification.
2. Summary of Special Characters
3. Summary of Columns with Inconsistent Units and Recommendations
"""
# Bold formatting
bold_start = "\033[1m"
bold_end = "\033[0m"
print(f"\n\n{bold_start} ---------------------- Quality Issues Inspection Part 2 ---------------------- {bold_end}\n")
print(f"{bold_start}Dataset:{bold_end} {dataset_name} \n\n")
# 1. Detect Special Characters: Separated into 2 functions to avoid the IOPub data rate exceeded warning
# 1.a. To display the summary
def detect_special_characters(df):
"""Detects special characters or formatting issues in object columns."""
special_characters_details = {}
special_characters_summary = {}
for col in df.select_dtypes(include=['object']).columns:
# Collect rows with special characters in the column
special_char_rows = []
unique_chars = set()
for value in df[col].dropna():
chars_in_value = re.findall(r'[^\w\s]', str(value))
if chars_in_value:
special_char_rows.append(value)
unique_chars.update(chars_in_value)
# Add to the dictionaries if special characters are found
if special_char_rows:
special_characters_details[col] = special_char_rows
special_characters_summary[col] = sorted(unique_chars)
# Remove columns without special characters
special_characters_details = {k: v for k, v in special_characters_details.items() if v}
special_characters_summary = {k: v for k, v in special_characters_summary.items() if v}
return special_characters_details, special_characters_summary
def display_special_characters_summary(special_characters_summary):
"""Displays the summary of special characters in a tabular format."""
if special_characters_summary:
summary_table = pd.DataFrame(
[{"Column": col, "Special Characters or Formatting Issues": ', '.join(chars)}
for col, chars in special_characters_summary.items()]
)
print("\n1. Summary of Special Characters:")
print(summary_table.to_string(index=False))
else:
print(f"\nNo special characters or formatting issues found in the dataset '{dataset_name}'.")
# To save the summary into csv files
def save_special_characters_summary(special_characters_summary, special_characters_details):
"""Saves the detailed and summary results to CSV files."""
if special_characters_details:
# Create and save the detailed CSV
max_len = max(len(v) for v in special_characters_details.values())
formatted_data = {col: v + [""] * (max_len - len(v)) for col, v in special_characters_details.items()}
special_characters_table = pd.DataFrame(formatted_data)
detailed_output_file = f"{dataset_name}_special_characters_details.csv"
special_characters_table.to_csv(detailed_output_file, index=False)
print(f"\nDetailed special characters (delete if not needed) saved to '{detailed_output_file}'.")
# Create and save the summary CSV
summary_table = pd.DataFrame(
[{"Column": col, "Special Characters or Formatting Issues": ', '.join(chars)}
for col, chars in special_characters_summary.items()]
)
summary_output_file = f"{dataset_name}_special_characters_summary.csv"
summary_table.to_csv(summary_output_file, index=False)
print(f"\nSummary of special characters (delete if not needed) saved to '{summary_output_file}'.")
# Detect and handle special characters
special_characters_details, special_characters_summary = detect_special_characters(df)
display_special_characters_summary(special_characters_summary)
save_special_characters_summary(special_characters_summary, special_characters_details)
# 2. Inconsistent Units
print("\n\n2. Columns with Inconsistent Units and Recommendations:")
def check_units(col):
"""Checks for alphabetic characters in a column."""
return df[col].str.contains(r'[a-zA-Z]', na=False) if df[col].dtype == 'object' else False
inconsistent_units_summary = []
for col in df.select_dtypes(include=['object']).columns:
inconsistent_units = df[check_units(col)]
if not inconsistent_units.empty:
# Determine if reengineering or reencoding is needed
if any(any(char.isdigit() for char in unit) for unit in set(inconsistent_units[col])):
recommendation = "Reengineer (Separate numeric values and units)"
else:
recommendation = "Reencode (Standardize categories)"
inconsistent_units_summary.append({"Column": col, "Recommendation": recommendation})
if inconsistent_units_summary:
inconsistent_units_table = pd.DataFrame(inconsistent_units_summary)
print(inconsistent_units_table.to_string(index=False))
# 3. Unique Categorical Values
print("\n\n3. Unique Categorical Values:")
unique_values_summary = {}
for col in df.select_dtypes(include=['object']).columns:
unique_values = df[col].dropna().unique()
if unique_values.size > 0:
unique_values_summary[col] = unique_values.tolist()
if unique_values_summary:
unique_values_table = pd.DataFrame(dict([(k, pd.Series(v)) for k, v in unique_values_summary.items()]))
output_file = f"{dataset_name}_unique_categorical_values.csv"
unique_values_table.to_csv(output_file, index=False)
print(f"\nUnique categorical values (delete if not needed) saved to '{output_file}'.")
else:
print(f"\nNo unique categorical values found in the dataset '{dataset_name}'.")
# 4. Analyze a dataset for data types, unexpected values, and generate a data dictionary
print(f"\n\n4. Analyzing Dataset: '{dataset_name}' for data types and classification")
# 4.a: Classify Column Type
def classify_column(col):
"""
Dynamically classify a column as 'categorical', 'ordinal', 'numeric', or 'other'.
"""
unique_vals = df[col].dropna().unique() # Drop NaN for unique values
if df[col].dtype == 'object':
return 'categorical'
elif df[col].dtype in ['int64', 'float64'] and len(unique_vals) <= 10:
return 'ordinal' # Numeric with limited unique values
elif df[col].dtype in ['int64', 'float64']:
return 'numeric'
else:
return 'other' # For columns that don’t fit the above types
# 4.b: Check If Data Type is Correct or Incorrect
def check_correct_data_type(col):
"""
Check if the actual data type of a column matches the expected type.
"""
expected_type = classify_column(col)
actual_dtype = df[col].dtype
# For categorical data
if expected_type == 'categorical':
if actual_dtype != 'object' and actual_dtype.name != 'category':
return 'Incorrect'
# For ordinal data (numeric with limited unique values)
elif expected_type == 'ordinal':
if actual_dtype not in ['int64', 'float64']:
return 'Incorrect'
# For numeric data
elif expected_type == 'numeric':
if actual_dtype not in ['int64', 'float64']:
return 'Incorrect'
# For other types (e.g., dates or mixed types)
elif expected_type == 'other':
try:
# Check if the column can be converted to datetime without errors
pd.to_datetime(df[col], errors='coerce')
except Exception:
return 'Incorrect'
# Correct if all checks pass
return 'Correct'
# 4.c: Generate Data Dictionary
data_dictionary = pd.DataFrame({
'Column': df.columns, # Column names
'Data Type': df.dtypes.astype(str), # Actual data type
'Classification': [classify_column(col) for col in df.columns], # Dynamic classification
'Correct/Incorrect Data Type': [check_correct_data_type(col) for col in df.columns], # Correctness check
'Unique Values': [df[col].nunique() for col in df.columns], # Count of unique values
'Sample Values': [df[col].dropna().unique()[:5] for col in df.columns] # First 5 unique non-NaN values
})
return data_dictionary
In [ ]:
In [6]:
def tidiness_issue_inspection(df, dataset_name):
"""
Inspects a DataFrame for tidiness issues, including:
1. Column headers as values
2. Wide vs. long format
3. Poorly named columns
4. Inconsistent Data Structure
5. Unnecessary Columns
6. Mixed Granularity
7. Repeated Index or Hierarchical Issues
Args:
df (pd.DataFrame): The DataFrame to analyze.
dataset_name (str): Name of the dataset.
Returns:
None
"""
# Bold formatting for terminal output
bold_start = "\033[1m"
bold_end = "\033[0m"
print(f"\n\n{bold_start} ---------------------- Tidiness Issues Inspection ---------------------- {bold_end}\n")
print(f"{bold_start}Dataset:{bold_end} {dataset_name} \n\n")
# 1. Column Headers as Values
print(f"{bold_start}1. Column Headers as Values:{bold_end}")
if any(char.isdigit() for col in df.columns for char in str(col)):
print(" Column headers may contain values (e.g., '2021-Q1').\n")
else:
print(" None\n")
# 2. Wide Format Instead of Long Format
print(f"{bold_start}2. Wide Format Instead of Long Format:{bold_end}")
wide_columns = [col for col in df.columns if any(char.isdigit() for char in str(col))]
if wide_columns:
print(f" Wide-format columns detected: {wide_columns}\n")
else:
print(" None\n")
# 3. Poorly Named Columns
print(f"{bold_start}3. Poorly Named Columns:{bold_end}")
poorly_named_columns = [
col for col in df.columns
if col.strip() == "" or col.isnumeric() or col.lower().startswith("unnamed")
]
if poorly_named_columns:
print(f" Poorly named columns detected: {poorly_named_columns}\n")
else:
print(" None\n")
# 4. Inconsistent Data Structure
print(f"{bold_start}4. Inconsistent Data Structure:{bold_end}")
inconsistent_rows = df.isnull().all(axis=1).sum()
if inconsistent_rows > 0:
print(f" Number of completely empty rows: {inconsistent_rows}\n")
else:
print(" None\n")
# 5. Unnecessary Columns
print(f"{bold_start}5. Unnecessary Columns:{bold_end}")
unnecessary_columns = [col for col in df.columns if df[col].nunique() == 1]
if unnecessary_columns:
print(f" Unnecessary columns detected: {unnecessary_columns}\n")
else:
print(" None\n")
# 6. Mixed Granularity
print(f"{bold_start}6. Mixed Granularity:{bold_end}")
potential_columns = []
# Iterate through each column to detect mixed granularity
for col in df.columns:
# Check for mixed date formats (e.g., YYYY-MM-DD and YYYY-MM)
if df[col].dtype == 'object': # Only check object/string columns
contains_full_date = df[col].str.contains(r'\d{4}-\d{2}-\d{2}', na=False).any()
contains_partial_date = df[col].str.contains(r'\d{4}-\d{2}', na=False).any()
if contains_full_date and contains_partial_date:
potential_columns.append(col)
# Check for numeric columns with inconsistent hierarchical group sizes
elif df[col].dtype in ['int64', 'float64']:
unique_group_sizes = df.groupby(col).size().nunique()
if unique_group_sizes > 1: # If group sizes vary, it may indicate mixed granularity
potential_columns.append(col)
# Output the results
if potential_columns:
print(f" Potential mixed granularity detected in columns: {potential_columns}\n")
else:
print(" None\n")
# 7. Repeated Index or Hierarchical Issues
print(f"{bold_start}7. Repeated Index or Hierarchical Issues:{bold_end}")
# Check for duplicate indices
if not df.index.is_unique:
print(" Index contains duplicates.\n")
duplicate_indices = df.index[df.index.duplicated()].unique()
print(f" Duplicate index values: {list(duplicate_indices)}\n")
print(" Consider resetting the index using `df.reset_index()` or aggregating rows by index.\n")
else:
print(" Index is unique.\n")
# Check if the DataFrame has a MultiIndex
if isinstance(df.index, pd.MultiIndex):
print(" Checking for missing levels in MultiIndex...\n")
# Convert MultiIndex to a DataFrame for inspection
index_frame = df.index.to_frame()
# Identify missing levels in each level of the MultiIndex
missing_levels = index_frame.isnull().any(axis=0)
if missing_levels.any():
missing_levels_list = missing_levels[missing_levels].index.tolist()
print(f" Missing levels detected in MultiIndex: {missing_levels_list}\n")
else:
print(" No missing levels in MultiIndex.\n")
else:
print(" Index is not a MultiIndex. No hierarchical issues detected.\n")
In [ ]:
In [7]:
# ==================================================
# Call functions for quality and tidiness inspection
# Dataset - county_data.csv
# ==================================================
# Note: Call these functions in this specific order
# Load the dataset - county_data
df = pd.read_csv("county_data.csv")
dataset_name = "county_data.csv"
# Call the analyze_dataset function
analyze_dataset(df, dataset_name)
# Check tidiness issues
tidiness_issue_inspection(df, dataset_name)
# Call the function for Quality Issues Inspection 1
quality_issue_inspection_part1(df, dataset_name)
# Call the function for Quality Issues Inspection 2
quality_issue_inspection_part2(df, dataset_name)
Dataset: county_data.csv Total number of rows: 3220 Total number of columns: 37 First 3 rows of the dataset:
| CensusId | State | County | TotalPop | Men | Women | Hispanic | White | Black | Native | Asian | Pacific | Citizen | Income | IncomeErr | IncomePerCap | IncomePerCapErr | Poverty | ChildPoverty | Professional | Service | Office | Construction | Production | Drive | Carpool | Transit | Walk | OtherTransp | WorkAtHome | MeanCommute | Employed | PrivateWork | PublicWork | SelfEmployed | FamilyWork | Unemployment | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 1001 | Alabama | Autauga | 55221 | 26745 | 28476 | 2.6 | 75.8 | 18.5 | 0.4 | 1.0 | 0.0 | 40725 | 51281.0 | 2391.0 | 24974 | 1080 | 12.9 | 18.6 | 33.2 | 17.0 | 24.2 | 8.6 | 17.1 | 87.5 | 8.8 | 0.1 | 0.5 | 1.3 | 1.8 | 26.5 | 23986 | 73.6 | 20.9 | 5.5 | 0.0 | 7.6 |
| 1 | 1003 | Alabama | Baldwin | 195121 | 95314 | 99807 | 4.5 | 83.1 | 9.5 | 0.6 | 0.7 | 0.0 | 147695 | 50254.0 | 1263.0 | 27317 | 711 | 13.4 | 19.2 | 33.1 | 17.7 | 27.1 | 10.8 | 11.2 | 84.7 | 8.8 | 0.1 | 1.0 | 1.4 | 3.9 | 26.4 | 85953 | 81.5 | 12.3 | 5.8 | 0.4 | 7.5 |
| 2 | 1005 | Alabama | Barbour | 26932 | 14497 | 12435 | 4.6 | 46.2 | 46.7 | 0.2 | 0.4 | 0.0 | 20714 | 32964.0 | 2973.0 | 16824 | 798 | 26.7 | 45.3 | 26.8 | 16.1 | 23.1 | 10.8 | 23.1 | 83.8 | 10.9 | 0.4 | 1.8 | 1.5 | 1.6 | 24.1 | 8597 | 71.8 | 20.8 | 7.3 | 0.1 | 17.6 |
All Columns in the Dataset: ['CensusId', 'State', 'County', 'TotalPop', 'Men', 'Women', 'Hispanic', 'White', 'Black', 'Native', 'Asian', 'Pacific', 'Citizen', 'Income', 'IncomeErr', 'IncomePerCap', 'IncomePerCapErr', 'Poverty', 'ChildPoverty', 'Professional', 'Service', 'Office', 'Construction', 'Production', 'Drive', 'Carpool', 'Transit', 'Walk', 'OtherTransp', 'WorkAtHome', 'MeanCommute', 'Employed', 'PrivateWork', 'PublicWork', 'SelfEmployed', 'FamilyWork', 'Unemployment'] No repeated columns found. ---------------------- Tidiness Issues Inspection ---------------------- Dataset: county_data.csv 1. Column Headers as Values: None 2. Wide Format Instead of Long Format: None 3. Poorly Named Columns: None 4. Inconsistent Data Structure: None 5. Unnecessary Columns: None 6. Mixed Granularity: Potential mixed granularity detected in columns: ['TotalPop', 'Men', 'Women', 'Hispanic', 'White', 'Black', 'Native', 'Asian', 'Pacific', 'Citizen', 'Income', 'IncomeErr', 'IncomePerCap', 'IncomePerCapErr', 'Poverty', 'ChildPoverty', 'Professional', 'Service', 'Office', 'Construction', 'Production', 'Drive', 'Carpool', 'Transit', 'Walk', 'OtherTransp', 'WorkAtHome', 'MeanCommute', 'Employed', 'PrivateWork', 'PublicWork', 'SelfEmployed', 'FamilyWork', 'Unemployment'] 7. Repeated Index or Hierarchical Issues: Index is unique. Index is not a MultiIndex. No hierarchical issues detected. ---------------------- Quality Issues Inspection Part 1 ---------------------- Dataset: county_data.csv 1. Missing Values: Missing Values Summary (Columns): Missing Values Percentage CensusId 0 0.00 State 0 0.00 County 0 0.00 TotalPop 0 0.00 Men 0 0.00 Women 0 0.00 Hispanic 0 0.00 White 0 0.00 Black 0 0.00 Native 0 0.00 Asian 0 0.00 Pacific 0 0.00 Citizen 0 0.00 Income 1 0.03 IncomeErr 1 0.03 IncomePerCap 0 0.00 IncomePerCapErr 0 0.00 Poverty 0 0.00 ChildPoverty 1 0.03 Professional 0 0.00 Service 0 0.00 Office 0 0.00 Construction 0 0.00 Production 0 0.00 Drive 0 0.00 Carpool 0 0.00 Transit 0 0.00 Walk 0 0.00 OtherTransp 0 0.00 WorkAtHome 0 0.00 MeanCommute 0 0.00 Employed 0 0.00 PrivateWork 0 0.00 PublicWork 0 0.00 SelfEmployed 0 0.00 FamilyWork 0 0.00 Unemployment 0 0.00 Missing Values Summary (Rows): Total rows with missing values: 2 (0.06%) Rows with high missing values (> 30%): 0 (0.00%) Rows with low missing values (0% - 30%): 2 (0.06%) Rows with no missing values: 3218 (99.94%) 2. Duplicate Rows: None 3. Invalid and/or Negative Values: None 4. Outliers Outliers detected in 'TotalPop': 431 rows Outliers detected in 'Men': 428 rows Outliers detected in 'Women': 431 rows Outliers detected in 'Hispanic': 407 rows Outliers detected in 'White': 130 rows Outliers detected in 'Black': 434 rows Outliers detected in 'Native': 438 rows Outliers detected in 'Asian': 323 rows Outliers detected in 'Pacific': 766 rows Outliers detected in 'Citizen': 421 rows Outliers detected in 'Income': 158 rows Outliers detected in 'IncomeErr': 175 rows Outliers detected in 'IncomePerCap': 152 rows Outliers detected in 'IncomePerCapErr': 180 rows Outliers detected in 'Poverty': 137 rows Outliers detected in 'ChildPoverty': 109 rows Outliers detected in 'Professional': 88 rows Outliers detected in 'Service': 101 rows Outliers detected in 'Office': 48 rows Outliers detected in 'Construction': 78 rows Outliers detected in 'Production': 30 rows Outliers detected in 'Drive': 150 rows Outliers detected in 'Carpool': 110 rows Outliers detected in 'Transit': 319 rows Outliers detected in 'Walk': 195 rows Outliers detected in 'OtherTransp': 184 rows Outliers detected in 'WorkAtHome': 165 rows Outliers detected in 'MeanCommute': 52 rows Outliers detected in 'Employed': 449 rows Outliers detected in 'PrivateWork': 100 rows Outliers detected in 'PublicWork': 120 rows Outliers detected in 'SelfEmployed': 165 rows Outliers detected in 'FamilyWork': 301 rows Outliers detected in 'Unemployment': 108 rows Columns with outliers: ['TotalPop', 'Men', 'Women', 'Hispanic', 'White', 'Black', 'Native', 'Asian', 'Pacific', 'Citizen', 'Income', 'IncomeErr', 'IncomePerCap', 'IncomePerCapErr', 'Poverty', 'ChildPoverty', 'Professional', 'Service', 'Office', 'Construction', 'Production', 'Drive', 'Carpool', 'Transit', 'Walk', 'OtherTransp', 'WorkAtHome', 'MeanCommute', 'Employed', 'PrivateWork', 'PublicWork', 'SelfEmployed', 'FamilyWork', 'Unemployment'] Total number of outliers across all columns: 7883 Percentage of total outliers in the dataset: 6.99% 5. Redundant Leading/Trailing Spaces: None 6. Column Entries with Mixed Formats: None 7. Non-Numeric Data in Numeric Columns: Column Non-Numeric Data Count Income [nan] 1 IncomeErr [nan] 1 ChildPoverty [nan] 1 8. Numeric Data in Non-Numeric Columns: None ---------------------- Quality Issues Inspection Part 2 ---------------------- Dataset: county_data.csv 1. Summary of Special Characters: Column Special Characters or Formatting Issues County ', -, . Detailed special characters (delete if not needed) saved to 'county_data.csv_special_characters_details.csv'. Summary of special characters (delete if not needed) saved to 'county_data.csv_special_characters_summary.csv'. 2. Columns with Inconsistent Units and Recommendations: Column Recommendation State Reencode (Standardize categories) County Reencode (Standardize categories) 3. Unique Categorical Values: Unique categorical values (delete if not needed) saved to 'county_data.csv_unique_categorical_values.csv'. 4. Analyzing Dataset: 'county_data.csv' for data types and classification
Out[7]:
| Column | Data Type | Classification | Correct/Incorrect Data Type | Unique Values | Sample Values | |
|---|---|---|---|---|---|---|
| CensusId | CensusId | int64 | numeric | Correct | 3220 | [1001, 1003, 1005, 1007, 1009] |
| State | State | object | categorical | Correct | 52 | [Alabama, Alaska, Arizona, Arkansas, California] |
| County | County | object | categorical | Correct | 1928 | [Autauga, Baldwin, Barbour, Bibb, Blount] |
| TotalPop | TotalPop | int64 | numeric | Correct | 3164 | [55221, 195121, 26932, 22604, 57710] |
| Men | Men | int64 | numeric | Correct | 3098 | [26745, 95314, 14497, 12073, 28512] |
| Women | Women | int64 | numeric | Correct | 3105 | [28476, 99807, 12435, 10531, 29198] |
| Hispanic | Hispanic | float64 | numeric | Correct | 466 | [2.6, 4.5, 4.6, 2.2, 8.6] |
| White | White | float64 | numeric | Correct | 721 | [75.8, 83.1, 46.2, 74.5, 87.9] |
| Black | Black | float64 | numeric | Correct | 487 | [18.5, 9.5, 46.7, 21.4, 1.5] |
| Native | Native | float64 | numeric | Correct | 183 | [0.4, 0.6, 0.2, 0.3, 1.2] |
| Asian | Asian | float64 | numeric | Correct | 131 | [1.0, 0.7, 0.4, 0.1, 0.2] |
| Pacific | Pacific | float64 | numeric | Correct | 27 | [0.0, 0.4, 0.1, 0.2, 1.1] |
| Citizen | Citizen | int64 | numeric | Correct | 3146 | [40725, 147695, 20714, 17495, 42345] |
| Income | Income | float64 | numeric | Correct | 3066 | [51281.0, 50254.0, 32964.0, 38678.0, 45813.0] |
| IncomeErr | IncomeErr | float64 | numeric | Correct | 2329 | [2391.0, 1263.0, 2973.0, 3995.0, 3141.0] |
| IncomePerCap | IncomePerCap | int64 | numeric | Correct | 2962 | [24974, 27317, 16824, 18431, 20532] |
| IncomePerCapErr | IncomePerCapErr | int64 | numeric | Correct | 1758 | [1080, 711, 798, 1618, 708] |
| Poverty | Poverty | float64 | numeric | Correct | 394 | [12.9, 13.4, 26.7, 16.8, 16.7] |
| ChildPoverty | ChildPoverty | float64 | numeric | Correct | 544 | [18.6, 19.2, 45.3, 27.9, 27.2] |
| Professional | Professional | float64 | numeric | Correct | 344 | [33.2, 33.1, 26.8, 21.5, 28.5] |
| Service | Service | float64 | numeric | Correct | 228 | [17.0, 17.7, 16.1, 17.9, 14.1] |
| Office | Office | float64 | numeric | Correct | 202 | [24.2, 27.1, 23.1, 17.8, 23.9] |
| Construction | Construction | float64 | numeric | Correct | 237 | [8.6, 10.8, 19.0, 13.5, 20.1] |
| Production | Production | float64 | numeric | Correct | 292 | [17.1, 11.2, 23.1, 23.7, 19.9] |
| Drive | Drive | float64 | numeric | Correct | 365 | [87.5, 84.7, 83.8, 83.2, 84.9] |
| Carpool | Carpool | float64 | numeric | Correct | 189 | [8.8, 10.9, 13.5, 11.2, 14.9] |
| Transit | Transit | float64 | numeric | Correct | 119 | [0.1, 0.4, 0.5, 0.7, 0.0] |
| Walk | Walk | float64 | numeric | Correct | 171 | [0.5, 1.0, 1.8, 0.6, 0.9] |
| OtherTransp | OtherTransp | float64 | numeric | Correct | 93 | [1.3, 1.4, 1.5, 0.4, 1.7] |
| WorkAtHome | WorkAtHome | float64 | numeric | Correct | 188 | [1.8, 3.9, 1.6, 0.7, 2.3] |
| MeanCommute | MeanCommute | float64 | numeric | Correct | 309 | [26.5, 26.4, 24.1, 28.8, 34.9] |
| Employed | Employed | int64 | numeric | Correct | 3098 | [23986, 85953, 8597, 8294, 22189] |
| PrivateWork | PrivateWork | float64 | numeric | Correct | 381 | [73.6, 81.5, 71.8, 76.8, 82.0] |
| PublicWork | PublicWork | float64 | numeric | Correct | 322 | [20.9, 12.3, 20.8, 16.1, 13.5] |
| SelfEmployed | SelfEmployed | float64 | numeric | Correct | 221 | [5.5, 5.8, 7.3, 6.7, 4.2] |
| FamilyWork | FamilyWork | float64 | numeric | Correct | 39 | [0.0, 0.4, 0.1, 0.2, 0.5] |
| Unemployment | Unemployment | float64 | numeric | Correct | 230 | [7.6, 7.5, 17.6, 8.3, 7.7] |
In [ ]:
After checking the data, these are the issues that need to be carefully checked and possibly need actions:
Missing values/NaN: Total missing values are 0.06% of the entire dataset. Decision: Remove these rows with missing values.
Special characters: Column 'County' has some special characters, but after inspection, there is no need to do any further actions as those are part of the county names
Columns 'State' and 'County' with reencoding advise: No need to reencode state and county since those are the predetermined data and should not be changed for data integrity.
Outliers: All columns contain outliers of 6.99% of the whole dataset. Decision: Leave them as for census data this large and a country this big, there will always be a significant disparity in numbers. 6.99% of total outliers are within the acceptable range for EDA. Another important note is that these outliers represent real phenomena and reflect diversity in data; they are relevant to the analysis as they provide meaningful insights into road construction and transportation challenges and their interconnecting factors.
Original Dataset has 52 entities (50 states, the District of Columbia, and Puerto Rico)
Remove Puerto Rico from the dataset as it is one of the territories and cannot vote for either the presidency and/or the congress. U.S. Territories cannot vote for both Congress and the presidency, and they do not have the same constitutional status, leading to reduced funding and rights. Furthermore, they are not fully incorporated into the U.S. and do not automatically receive all constitutional rights and benefits. Lastly, Federal funding formulas often treat territories differently, resulting in capped funding specially in healthcare and highways, and severe delays in receiving relief funds.
Remove the District of Columbia from the dataset as it is a federal district, not a state. Although D.C. has three college electoral votes in the presidency, they do not have representation in Congress, they do not have Senators, and has one non-voting delegate in the House of Representatives, who can speak, propose legislation, and serve on committees but cannot vote on final bills. These reasons result in funding disparities and limited autonomy over budget and laws.
In [ ]:
In [8]:
# Drop missing/nan rows with missing values since it is only 0.06%
# Process the data for defining columns, adding columns, calculating transportation score
# Drop rows containing the District of Columbia and Puerto Rico
# Load the dataset - county_data
df = pd.read_csv("county_data.csv")
# ----------- Adding columns from here -----------
# These are the columns that I think are necessary for me to conduct the analysis.
# Define coastal states
coastal_states = ['Maine', 'New Hampshire', 'Massachusetts', 'Rhode Island', 'Connecticut', 'New York',
'New Jersey', 'Delaware', 'Maryland', 'Virginia', 'North Carolina', 'South Carolina',
'Georgia', 'Florida', 'Alabama', 'Mississippi', 'Louisiana', 'Texas', 'California',
'Oregon', 'Washington', 'Alaska', 'Hawaii']
# Add "IsCoastal" column for coastal/inland classification
df['IsCoastal'] = df['State'].apply(lambda x: 'Coastal' if x in coastal_states else 'Inland')
# Define U.S. Census Regions and Divisions
print("\nProcessing: adding regions and divisions columns...")
regions_divisions = {
'Northeast': {
'New England': ['Connecticut', 'Maine', 'Massachusetts', 'New Hampshire', 'Rhode Island', 'Vermont'],
'Middle Atlantic': ['New Jersey', 'New York', 'Pennsylvania']
},
'Midwest': {
'East North Central': ['Indiana', 'Illinois', 'Michigan', 'Ohio', 'Wisconsin'],
'West North Central': ['Iowa', 'Kansas', 'Minnesota', 'Missouri', 'Nebraska', 'North Dakota', 'South Dakota']
},
'South': {
'South Atlantic': ['Delaware', 'District of Columbia', 'Florida', 'Georgia', 'Maryland', 'North Carolina',
'South Carolina', 'Virginia', 'West Virginia'],
'East South Central': ['Alabama', 'Kentucky', 'Mississippi', 'Tennessee'],
'West South Central': ['Arkansas', 'Louisiana', 'Oklahoma', 'Texas']
},
'West': {
'Mountain': ['Arizona', 'Colorado', 'Idaho', 'Montana', 'Nevada', 'New Mexico', 'Utah', 'Wyoming'],
'Pacific': ['Alaska', 'California', 'Hawaii', 'Oregon', 'Washington']
}
}
# Flatten the dictionary to map states to regions and divisions
state_to_region_division = {}
for region, divisions in regions_divisions.items():
for division, states in divisions.items():
for state in states:
state_to_region_division[state] = {'Region': region, 'Division': division}
# Add Region and Division Columns to the df
def add_region_division_columns(df):
"""
Adds Region and Division columns to the DataFrame based on U.S. Census classifications.
"""
df['Region'] = df['State'].map(lambda state: state_to_region_division.get(state, {}).get('Region'))
df['Division'] = df['State'].map(lambda state: state_to_region_division.get(state, {}).get('Division'))
return df
# Add Region and Division columns to county-level data
df = add_region_division_columns(df)
# ----------- Adding columns ENDS here -----------
# Calculate Transportation Score
df['TransportationScore'] = (
0.4 * df['Transit'] + # Public transit usage (weighted higher than others)
0.2 * df['Walk'] + # Walkability
0.2 * df['OtherTransp'] + # Other transportation options (e.g., bikes, rideshare)
0.2 * df['Carpool'] + # Shared private transportation
0.3 * df['Drive'] - # Private driving is considered positive since other counties do not have access to public transportation
0.5 * df['MeanCommute'] # Longer commute times are penalized
)
# Drop rows for District of Columbia and Puerto Rico
print("\nDropping District of Columbia and Puerto Rico...")
df = df[~df['State'].isin(['District of Columbia', 'Puerto Rico'])]
# Drop rows with missing values
print("\nProcessing missing rows...")
df = df.dropna()
# Save cleaned and preprocessed data
df.to_csv("CountyData_Processed_With_Added_Columns.csv", index=False)
print("\nCounty data with IsCoastal, Region, Division, TransportationScore columns saved as 'CountyData_Processed_With_Added_Columns.csv'.\n")
# Load the newly saved dataset
new_df1 = pd.read_csv("CountyData_Processed_With_Added_Columns.csv")
# Display the newly saved dataset
print("\nFirst 5 rows of the newly saved dataset 'CountyData_Processed_With_Added_Columns.csv':\n")
display(new_df1.head())
# Display all column names
print("\n\nAll Columns in the Dataset:")
print(new_df1.columns.tolist())
Processing: adding regions and divisions columns... Dropping District of Columbia and Puerto Rico... Processing missing rows... County data with IsCoastal, Region, Division, TransportationScore columns saved as 'CountyData_Processed_With_Added_Columns.csv'. First 5 rows of the newly saved dataset 'CountyData_Processed_With_Added_Columns.csv':
| CensusId | State | County | TotalPop | Men | Women | Hispanic | White | Black | Native | Asian | Pacific | Citizen | Income | IncomeErr | IncomePerCap | IncomePerCapErr | Poverty | ChildPoverty | Professional | Service | Office | Construction | Production | Drive | Carpool | Transit | Walk | OtherTransp | WorkAtHome | MeanCommute | Employed | PrivateWork | PublicWork | SelfEmployed | FamilyWork | Unemployment | IsCoastal | Region | Division | TransportationScore | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 1001 | Alabama | Autauga | 55221 | 26745 | 28476 | 2.6 | 75.8 | 18.5 | 0.4 | 1.0 | 0.0 | 40725 | 51281.0 | 2391.0 | 24974 | 1080 | 12.9 | 18.6 | 33.2 | 17.0 | 24.2 | 8.6 | 17.1 | 87.5 | 8.8 | 0.1 | 0.5 | 1.3 | 1.8 | 26.5 | 23986 | 73.6 | 20.9 | 5.5 | 0.0 | 7.6 | Coastal | South | East South Central | 15.16 |
| 1 | 1003 | Alabama | Baldwin | 195121 | 95314 | 99807 | 4.5 | 83.1 | 9.5 | 0.6 | 0.7 | 0.0 | 147695 | 50254.0 | 1263.0 | 27317 | 711 | 13.4 | 19.2 | 33.1 | 17.7 | 27.1 | 10.8 | 11.2 | 84.7 | 8.8 | 0.1 | 1.0 | 1.4 | 3.9 | 26.4 | 85953 | 81.5 | 12.3 | 5.8 | 0.4 | 7.5 | Coastal | South | East South Central | 14.49 |
| 2 | 1005 | Alabama | Barbour | 26932 | 14497 | 12435 | 4.6 | 46.2 | 46.7 | 0.2 | 0.4 | 0.0 | 20714 | 32964.0 | 2973.0 | 16824 | 798 | 26.7 | 45.3 | 26.8 | 16.1 | 23.1 | 10.8 | 23.1 | 83.8 | 10.9 | 0.4 | 1.8 | 1.5 | 1.6 | 24.1 | 8597 | 71.8 | 20.8 | 7.3 | 0.1 | 17.6 | Coastal | South | East South Central | 16.09 |
| 3 | 1007 | Alabama | Bibb | 22604 | 12073 | 10531 | 2.2 | 74.5 | 21.4 | 0.4 | 0.1 | 0.0 | 17495 | 38678.0 | 3995.0 | 18431 | 1618 | 16.8 | 27.9 | 21.5 | 17.9 | 17.8 | 19.0 | 23.7 | 83.2 | 13.5 | 0.5 | 0.6 | 1.5 | 0.7 | 28.8 | 8294 | 76.8 | 16.1 | 6.7 | 0.4 | 8.3 | Coastal | South | East South Central | 13.88 |
| 4 | 1009 | Alabama | Blount | 57710 | 28512 | 29198 | 8.6 | 87.9 | 1.5 | 0.3 | 0.1 | 0.0 | 42345 | 45813.0 | 3141.0 | 20532 | 708 | 16.7 | 27.2 | 28.5 | 14.1 | 23.9 | 13.5 | 19.9 | 84.9 | 11.2 | 0.4 | 0.9 | 0.4 | 2.3 | 34.9 | 22189 | 82.0 | 13.5 | 4.2 | 0.4 | 7.7 | Coastal | South | East South Central | 10.68 |
All Columns in the Dataset: ['CensusId', 'State', 'County', 'TotalPop', 'Men', 'Women', 'Hispanic', 'White', 'Black', 'Native', 'Asian', 'Pacific', 'Citizen', 'Income', 'IncomeErr', 'IncomePerCap', 'IncomePerCapErr', 'Poverty', 'ChildPoverty', 'Professional', 'Service', 'Office', 'Construction', 'Production', 'Drive', 'Carpool', 'Transit', 'Walk', 'OtherTransp', 'WorkAtHome', 'MeanCommute', 'Employed', 'PrivateWork', 'PublicWork', 'SelfEmployed', 'FamilyWork', 'Unemployment', 'IsCoastal', 'Region', 'Division', 'TransportationScore']
In [ ]:
In [9]:
# Load the dataset - processed county_data
df = pd.read_csv("CountyData_Processed_With_Added_Columns.csv")
# Aggregate Data from County to State Level
def aggregate_to_state_level(df):
"""
Aggregates county-level data to state level.
Uses population-weighted averages for metrics.
"""
# Add Region and Division columns before aggregation
df = add_region_division_columns(df)
# Aggregating only necessary columns
state_level = df.groupby(['State', 'Region', 'Division']).agg(
TotalPop=('TotalPop', 'sum'), # Sum the population for each state
Transit=('Transit', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()),
Walk=('Walk', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()),
OtherTransp=('OtherTransp', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()),
Income=('Income', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()),
Poverty=('Poverty', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()),
ChildPoverty=('ChildPoverty', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()), ### Just in case I will need this column.
Construction=('Construction', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()),
MeanCommute=('MeanCommute', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()),
TransportationScore=('TransportationScore', lambda x: (x * df.loc[x.index, 'TotalPop']).sum() / df.loc[x.index, 'TotalPop'].sum()),
IsCoastal=('IsCoastal', lambda x: x.mode()[0] if not x.mode().empty else None) # Most common coastal/inland value
).reset_index()
# Add StateRank based on TransportationScore
state_level['StateRank'] = state_level['TransportationScore'].rank(ascending=False, method='dense').astype(int)
# Sort by StateRank for readability
state_level = state_level.sort_values(by='StateRank')
return state_level
# Aggregate data to state level
state_level_data = aggregate_to_state_level(df)
# Save aggregated state-level data
state_level_data.to_csv("Aggregated_StateData_with_critical_columns.csv", index=False)
print("\nState-level data with regions and divisions saved as 'Aggregated_StateData_with_critical_columns.csv'.")
# Load the newly saved dataset
new_df2 = pd.read_csv("Aggregated_StateData_with_critical_columns.csv")
# Display the newly saved dataset
print("\nFirst 5 rows of the newly saved dataset 'Aggregated_StateData_with_critical_columns.csv':\n")
display(new_df2.head())
# Display all column names
print("\n\nAll Columns in the Dataset:")
print(new_df2.columns.tolist())
State-level data with regions and divisions saved as 'Aggregated_StateData_with_critical_columns.csv'. First 5 rows of the newly saved dataset 'Aggregated_StateData_with_critical_columns.csv':
| State | Region | Division | TotalPop | Transit | Walk | OtherTransp | Income | Poverty | ChildPoverty | Construction | MeanCommute | TransportationScore | IsCoastal | StateRank | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | North Dakota | Midwest | West North Central | 721640 | 0.452494 | 3.988160 | 1.178630 | 57844.988889 | 11.469244 | 12.944757 | 13.252200 | 17.243660 | 18.430295 | Inland | 1 |
| 1 | South Dakota | Midwest | West North Central | 843190 | 0.544100 | 4.101369 | 1.520920 | 51216.943560 | 14.139960 | 17.684579 | 11.046249 | 16.952293 | 18.333544 | Inland | 2 |
| 2 | Nebraska | Midwest | West North Central | 1869365 | 0.722383 | 2.839618 | 1.330409 | 53793.719829 | 12.749011 | 17.051571 | 10.227180 | 18.251230 | 18.262106 | Inland | 3 |
| 3 | Kansas | Midwest | West North Central | 2892987 | 0.475756 | 2.359241 | 1.333645 | 53995.056227 | 13.625083 | 17.825993 | 10.069354 | 19.094261 | 17.952866 | Inland | 4 |
| 4 | Iowa | Midwest | West North Central | 3093526 | 1.091435 | 3.496906 | 1.528807 | 53796.169100 | 12.527552 | 15.793186 | 9.435677 | 18.885953 | 17.944372 | Inland | 5 |
All Columns in the Dataset: ['State', 'Region', 'Division', 'TotalPop', 'Transit', 'Walk', 'OtherTransp', 'Income', 'Poverty', 'ChildPoverty', 'Construction', 'MeanCommute', 'TransportationScore', 'IsCoastal', 'StateRank']
In [ ]: