Pandas in Jupyter: The Dynamic Duo That Makes Data Analysis Feel Like Magic

 1import pandas as pd
 2import numpy as np
 3import matplotlib.pyplot as plt
 4import seaborn as sns
 5
 6# Set up for optimal display
 7pd.set_option('display.max_columns', None)
 8pd.set_option('display.max_rows', 20)
 9pd.set_option('display.width', None)
10%matplotlib inline
11
12# Load data with immediate feedback
13print("Loading sales data...")
14df = pd.read_csv('sales_data.csv')
15
16# Instant overview - this is where the magic begins
17print(f"📊 Dataset loaded: {df.shape[0]:,} rows × {df.shape[1]} columns")
18print(f"💾 Memory usage: {df.memory_usage(deep=True).sum() / 1024**2:.1f} MB")
19print(f"📅 Date range: {df['date'].min()} to {df['date'].max()}")
20
21# See your data immediately
22df.head()

 1def explore_data_quality(df):
 2    """Interactive data quality assessment."""
 3    
 4    print("🔍 DATA QUALITY ASSESSMENT")
 5    print("=" * 50)
 6    
 7    # Basic info with visual formatting
 8    print(f"📊 Shape: {df.shape[0]:,} rows × {df.shape[1]} columns")
 9    print(f"🧮 Data types:")
10    type_counts = df.dtypes.value_counts()
11    for dtype, count in type_counts.items():
12        print(f"   {dtype}: {count} columns")
13    
14    print("\n🕳️ Missing Values:")
15    missing = df.isnull().sum()
16    missing_pct = (missing / len(df)) * 100
17    
18    missing_summary = pd.DataFrame({
19        'Column': missing.index,
20        'Missing Count': missing.values,
21        'Missing %': missing_pct.values
22    })
23    
24    # Only show columns with missing data
25    has_missing = missing_summary[missing_summary['Missing Count'] > 0]
26    
27    if len(has_missing) > 0:
28        print(has_missing.sort_values('Missing Count', ascending=False).to_string(index=False))
29        
30        # Visual representation
31        plt.figure(figsize=(12, 6))
32        has_missing.plot(x='Column', y='Missing %', kind='bar')
33        plt.title('Missing Data by Column')
34        plt.xticks(rotation=45)
35        plt.tight_layout()
36        plt.show()
37    else:
38        print("✅ No missing values found!")
39    
40    print("\n🔢 Numerical Columns Summary:")
41    numeric_cols = df.select_dtypes(include=[np.number]).columns
42    if len(numeric_cols) > 0:
43        return df[numeric_cols].describe()
44    else:
45        print("No numerical columns found")
46
47# Use the function
48summary_stats = explore_data_quality(df)
49display(summary_stats)  # Pretty display in Jupyter

 1# Start broad, then narrow down based on what you discover
 2print("🔍 Progressive Data Exploration")
 3
 4# Step 1: Overall view
 5print(f"Total records: {len(df):,}")
 6print(f"Unique products: {df['product'].nunique()}")
 7print(f"Date range: {df['date'].min()} to {df['date'].max()}")
 8
 9# Step 2: Look at distributions
10print("\n📊 Top 10 Products by Sales Volume:")
11top_products = df.groupby('product')['sales'].sum().sort_values(ascending=False).head(10)
12display(top_products)
13
14# Step 3: Dive deeper into the top product
15top_product = top_products.index[0]
16print(f"\n🔍 Deep dive: {top_product}")
17
18top_product_data = df[df['product'] == top_product]
19print(f"Records for {top_product}: {len(top_product_data):,}")
20
21# Step 4: Analyze trends for this product
22plt.figure(figsize=(15, 5))
23
24# Sales over time
25plt.subplot(1, 3, 1)
26monthly_sales = top_product_data.groupby(pd.to_datetime(top_product_data['date']).dt.to_period('M'))['sales'].sum()
27monthly_sales.plot()
28plt.title(f'{top_product} - Monthly Sales')
29plt.xticks(rotation=45)
30
31# Regional distribution
32plt.subplot(1, 3, 2)
33regional_sales = top_product_data.groupby('region')['sales'].sum()
34regional_sales.plot(kind='bar')
35plt.title(f'{top_product} - By Region')
36plt.xticks(rotation=45)
37
38# Sales distribution
39plt.subplot(1, 3, 3)
40top_product_data['sales'].hist(bins=30, alpha=0.7)
41plt.title(f'{top_product} - Sales Distribution')
42
43plt.tight_layout()
44plt.show()
45
46# Step 5: Statistical insights
47print(f"\n📈 {top_product} Statistics:")
48stats = top_product_data['sales'].describe()
49print(f"Average sale: ${stats['mean']:.2f}")
50print(f"Median sale: ${stats['50%']:.2f}")
51print(f"Top 5% threshold: ${stats.quantile(0.95):.2f}")
52
53# Find outliers
54outlier_threshold = stats.quantile(0.95)
55outliers = top_product_data[top_product_data['sales'] > outlier_threshold]
56print(f"High-value transactions (>{stats.quantile(0.95):.0f}): {len(outliers)}")
57
58if len(outliers) > 0:
59    print("\nTop 5 highest transactions:")
60    display(outliers.nlargest(5, 'sales')[['date', 'region', 'sales']])

 1# Compare multiple dimensions simultaneously
 2def compare_categories(df, metric='sales'):
 3    """Compare performance across different categories."""
 4    
 5    categories = df['category'].unique()
 6    
 7    print(f"📊 Comparing {len(categories)} categories by {metric}")
 8    print("=" * 50)
 9    
10    # Create comparison dataframe
11    comparison_data = []
12    
13    for category in categories:
14        cat_data = df[df['category'] == category]
15        
16        stats = {
17            'Category': category,
18            'Total Sales': cat_data[metric].sum(),
19            'Avg Sale': cat_data[metric].mean(),
20            'Median Sale': cat_data[metric].median(),
21            'Std Dev': cat_data[metric].std(),
22            'Records': len(cat_data),
23            'Unique Products': cat_data['product'].nunique()
24        }
25        comparison_data.append(stats)
26    
27    comparison_df = pd.DataFrame(comparison_data)
28    comparison_df = comparison_df.sort_values('Total Sales', ascending=False)
29    
30    # Display formatted results
31    print("Category Performance Summary:")
32    display(comparison_df.style.format({
33        'Total Sales': '${:,.0f}',
34        'Avg Sale': '${:.2f}',
35        'Median Sale': '${:.2f}',
36        'Std Dev': '${:.2f}',
37        'Records': '{:,}'
38    }))
39    
40    # Visual comparison
41    fig, axes = plt.subplots(2, 2, figsize=(16, 12))
42    
43    # Total sales by category
44    axes[0, 0].bar(comparison_df['Category'], comparison_df['Total Sales'])
45    axes[0, 0].set_title('Total Sales by Category')
46    axes[0, 0].tick_params(axis='x', rotation=45)
47    
48    # Average sale by category
49    axes[0, 1].bar(comparison_df['Category'], comparison_df['Avg Sale'])
50    axes[0, 1].set_title('Average Sale by Category')
51    axes[0, 1].tick_params(axis='x', rotation=45)
52    
53    # Record count by category
54    axes[1, 0].bar(comparison_df['Category'], comparison_df['Records'])
55    axes[1, 0].set_title('Number of Records by Category')
56    axes[1, 0].tick_params(axis='x', rotation=45)
57    
58    # Sales distribution comparison (box plot)
59    category_data = [df[df['category'] == cat][metric] for cat in categories]
60    axes[1, 1].boxplot(category_data, labels=categories)
61    axes[1, 1].set_title('Sales Distribution by Category')
62    axes[1, 1].tick_params(axis='x', rotation=45)
63    
64    plt.tight_layout()
65    plt.show()
66    
67    return comparison_df
68
69# Use the function
70category_comparison = compare_categories(df)

 1# Convert to datetime and set up time series analysis
 2df['date'] = pd.to_datetime(df['date'])
 3df = df.sort_values('date')
 4
 5# Create time-based aggregations
 6def analyze_time_trends(df):
 7    """Interactive time series analysis."""
 8    
 9    print("📅 TIME SERIES ANALYSIS")
10    print("=" * 40)
11    
12    # Daily aggregation
13    daily_sales = df.groupby('date')['sales'].agg(['sum', 'count', 'mean']).reset_index()
14    daily_sales.columns = ['date', 'total_sales', 'num_transactions', 'avg_sale']
15    
16    # Weekly aggregation
17    weekly_sales = df.groupby(pd.Grouper(key='date', freq='W'))['sales'].agg(['sum', 'count', 'mean']).reset_index()
18    weekly_sales.columns = ['week', 'total_sales', 'num_transactions', 'avg_sale']
19    
20    # Monthly aggregation
21    monthly_sales = df.groupby(pd.Grouper(key='date', freq='M'))['sales'].agg(['sum', 'count', 'mean']).reset_index()
22    monthly_sales.columns = ['month', 'total_sales', 'num_transactions', 'avg_sale']
23    
24    # Create comprehensive time series plots
25    plt.figure(figsize=(18, 12))
26    
27    # Daily trends
28    plt.subplot(3, 2, 1)
29    plt.plot(daily_sales['date'], daily_sales['total_sales'])
30    plt.title('Daily Total Sales')
31    plt.xticks(rotation=45)
32    
33    plt.subplot(3, 2, 2)
34    plt.plot(daily_sales['date'], daily_sales['num_transactions'])
35    plt.title('Daily Transaction Count')
36    plt.xticks(rotation=45)
37    
38    # Weekly trends with moving average
39    plt.subplot(3, 2, 3)
40    plt.plot(weekly_sales['week'], weekly_sales['total_sales'], label='Weekly Sales', alpha=0.7)
41    # 4-week moving average
42    weekly_ma = weekly_sales['total_sales'].rolling(window=4).mean()
43    plt.plot(weekly_sales['week'], weekly_ma, label='4-Week Moving Average', color='red', linewidth=2)
44    plt.title('Weekly Sales with Moving Average')
45    plt.legend()
46    plt.xticks(rotation=45)
47    
48    plt.subplot(3, 2, 4)
49    plt.plot(weekly_sales['week'], weekly_sales['avg_sale'])
50    plt.title('Weekly Average Sale Amount')
51    plt.xticks(rotation=45)
52    
53    # Monthly trends
54    plt.subplot(3, 2, 5)
55    plt.bar(monthly_sales['month'], monthly_sales['total_sales'])
56    plt.title('Monthly Total Sales')
57    plt.xticks(rotation=45)
58    
59    # Seasonal analysis
60    plt.subplot(3, 2, 6)
61    df['month_name'] = df['date'].dt.month_name()
62    seasonal = df.groupby('month_name')['sales'].sum().reindex([
63        'January', 'February', 'March', 'April', 'May', 'June',
64        'July', 'August', 'September', 'October', 'November', 'December'
65    ])
66    seasonal.plot(kind='bar')
67    plt.title('Seasonal Sales Pattern')
68    plt.xticks(rotation=45)
69    
70    plt.tight_layout()
71    plt.show()
72    
73    # Statistical insights
74    print("\n📊 Time Series Insights:")
75    
76    # Growth analysis
77    first_month = monthly_sales['total_sales'].iloc[0]
78    last_month = monthly_sales['total_sales'].iloc[-1]
79    growth_rate = ((last_month - first_month) / first_month) * 100
80    
81    print(f"📈 Overall growth: {growth_rate:.1f}%")
82    print(f"💰 Best month: {monthly_sales.loc[monthly_sales['total_sales'].idxmax(), 'month'].strftime('%B %Y')} (${monthly_sales['total_sales'].max():,.0f})")
83    print(f"📉 Worst month: {monthly_sales.loc[monthly_sales['total_sales'].idxmin(), 'month'].strftime('%B %Y')} (${monthly_sales['total_sales'].min():,.0f})")
84    
85    # Volatility analysis
86    daily_volatility = daily_sales['total_sales'].std() / daily_sales['total_sales'].mean()
87    print(f"📊 Daily sales volatility: {daily_volatility:.2%}")
88    
89    return daily_sales, weekly_sales, monthly_sales
90
91# Run the analysis
92daily, weekly, monthly = analyze_time_trends(df)
93
94# Show sample of results
95print("\nSample of daily sales data:")
96display(daily.head())

 1# Multi-level grouping with custom aggregations
 2def advanced_grouping_analysis(df):
 3    """Demonstrate complex grouping patterns."""
 4    
 5    print("🎯 ADVANCED GROUPING ANALYSIS")
 6    print("=" * 45)
 7    
 8    # Multi-level grouping
 9    regional_product_analysis = df.groupby(['region', 'category', 'product']).agg({
10        'sales': ['sum', 'mean', 'std', 'count'],
11        'date': ['min', 'max']
12    }).round(2)
13    
14    # Flatten column names
15    regional_product_analysis.columns = ['_'.join(col).strip() for col in regional_product_analysis.columns]
16    regional_product_analysis = regional_product_analysis.reset_index()
17    
18    print("Top 10 Product-Region combinations by total sales:")
19    top_combinations = regional_product_analysis.nlargest(10, 'sales_sum')
20    display(top_combinations[['region', 'category', 'product', 'sales_sum', 'sales_mean', 'sales_count']])
21    
22    # Custom aggregation functions
23    def sales_metrics(series):
24        return pd.Series({
25            'total': series.sum(),
26            'avg': series.mean(),
27            'median': series.median(),
28            'q75': series.quantile(0.75),
29            'q25': series.quantile(0.25),
30            'cv': series.std() / series.mean(),  # Coefficient of variation
31            'days_active': len(series)
32        })
33    
34    # Apply custom aggregation
35    print("\n📊 Custom Sales Metrics by Category:")
36    category_metrics = df.groupby('category')['sales'].apply(sales_metrics).round(3)
37    display(category_metrics)
38    
39    # Pivot table analysis
40    print("\n🔄 Pivot Analysis - Sales by Region and Month:")
41    df['year_month'] = df['date'].dt.to_period('M')
42    pivot_sales = df.pivot_table(
43        values='sales', 
44        index='region', 
45        columns='year_month', 
46        aggfunc='sum', 
47        fill_value=0
48    )
49    
50    # Show recent months only
51    recent_columns = pivot_sales.columns[-6:]  # Last 6 months
52    recent_pivot = pivot_sales[recent_columns]
53    display(recent_pivot)
54    
55    # Heatmap visualization
56    plt.figure(figsize=(12, 6))
57    sns.heatmap(recent_pivot, annot=True, fmt='.0f', cmap='YlOrRd')
58    plt.title('Sales Heatmap: Region vs Month')
59    plt.tight_layout()
60    plt.show()
61    
62    return regional_product_analysis, category_metrics, pivot_sales
63
64# Execute analysis
65regional_analysis, metrics, pivot_data = advanced_grouping_analysis(df)

 1# Feature engineering in interactive context
 2def create_features(df):
 3    """Interactive feature engineering session."""
 4    
 5    print("🔧 FEATURE ENGINEERING")
 6    print("=" * 30)
 7    
 8    # Start with a copy
 9    df_features = df.copy()
10    
11    # Time-based features
12    print("📅 Creating time-based features...")
13    df_features['year'] = df_features['date'].dt.year
14    df_features['month'] = df_features['date'].dt.month
15    df_features['day_of_week'] = df_features['date'].dt.dayofweek
16    df_features['day_name'] = df_features['date'].dt.day_name()
17    df_features['is_weekend'] = df_features['day_of_week'].isin([5, 6])
18    df_features['quarter'] = df_features['date'].dt.quarter
19    df_features['week_of_year'] = df_features['date'].dt.isocalendar().week
20    
21    # Sales-based features
22    print("💰 Creating sales-based features...")
23    
24    # Sales categories
25    df_features['sales_category'] = pd.cut(
26        df_features['sales'], 
27        bins=[0, 10, 50, 200, float('inf')], 
28        labels=['Low', 'Medium', 'High', 'Premium']
29    )
30    
31    # Running statistics (requires sorted data)
32    df_features = df_features.sort_values(['product', 'date'])
33    
34    # Product-level rolling statistics
35    df_features['sales_7day_avg'] = df_features.groupby('product')['sales'].transform(
36        lambda x: x.rolling(window=7, min_periods=1).mean()
37    )
38    
39    df_features['sales_30day_avg'] = df_features.groupby('product')['sales'].transform(
40        lambda x: x.rolling(window=30, min_periods=1).mean()
41    )
42    
43    # Lag features
44    df_features['sales_prev_day'] = df_features.groupby('product')['sales'].shift(1)
45    df_features['sales_prev_week'] = df_features.groupby('product')['sales'].shift(7)
46    
47    # Growth rates
48    df_features['sales_growth_daily'] = (
49        (df_features['sales'] - df_features['sales_prev_day']) / df_features['sales_prev_day'] * 100
50    )
51    
52    # Relative to product average
53    product_means = df_features.groupby('product')['sales'].transform('mean')
54    df_features['sales_vs_product_avg'] = (df_features['sales'] - product_means) / product_means * 100
55    
56    # Relative to category average
57    category_means = df_features.groupby('category')['sales'].transform('mean')
58    df_features['sales_vs_category_avg'] = (df_features['sales'] - category_means) / category_means * 100
59    
60    # Regional market share
61    regional_totals = df_features.groupby(['region', 'date'])['sales'].transform('sum')
62    df_features['regional_market_share'] = df_features['sales'] / regional_totals * 100
63    
64    print("✅ Feature engineering complete!")
65    print(f"📊 Original columns: {len(df.columns)}")
66    print(f"📊 New columns: {len(df_features.columns)}")
67    print(f"📊 Added features: {len(df_features.columns) - len(df.columns)}")
68    
69    # Show sample of new features
70    print("\n🔍 Sample of new features:")
71    feature_columns = [
72        'product', 'date', 'sales', 'day_name', 'is_weekend', 
73        'sales_category', 'sales_7day_avg', 'sales_growth_daily', 'regional_market_share'
74    ]
75    display(df_features[feature_columns].head(10))
76    
77    # Feature correlation analysis
78    print("\n🔗 Feature Correlations with Sales:")
79    numeric_features = df_features.select_dtypes(include=[np.number]).columns
80    correlations = df_features[numeric_features].corr()['sales'].sort_values(key=abs, ascending=False)
81    
82    # Show top correlations
83    top_correlations = correlations[correlations.index != 'sales'].head(10)
84    for feature, corr in top_correlations.items():
85        direction = "📈" if corr > 0 else "📉"
86        print(f"  {direction} {feature}: {corr:.3f}")
87    
88    return df_features
89
90# Apply feature engineering
91df_enhanced = create_features(df)

 1# Optimize pandas display for Jupyter
 2def setup_pandas_display():
 3    """Configure pandas for optimal Jupyter display."""
 4    
 5    # Display options
 6    pd.set_option('display.max_columns', None)  # Show all columns
 7    pd.set_option('display.max_rows', 25)       # Reasonable row limit
 8    pd.set_option('display.width', None)        # No width limit
 9    pd.set_option('display.max_colwidth', 50)   # Reasonable column width
10    
11    # Precision for floats
12    pd.set_option('display.precision', 3)
13    
14    # Formatting
15    pd.set_option('display.float_format', '{:.3f}'.format)
16    
17    print("✅ Pandas display optimized for Jupyter!")
18
19# Custom display functions
20def display_with_info(df, title="DataFrame"):
21    """Enhanced display function for DataFrames."""
22    
23    print(f"📊 {title}")
24    print("=" * len(title))
25    print(f"Shape: {df.shape[0]:,} rows × {df.shape[1]} columns")
26    print(f"Memory: {df.memory_usage(deep=True).sum() / 1024**2:.1f} MB")
27    
28    if df.shape[0] > 0:
29        print(f"Date range: {df['date'].min()} to {df['date'].max()}" if 'date' in df.columns else "")
30        print()
31        display(df.head())
32        
33        if df.shape[0] > 5:
34            print(f"\n... and {df.shape[0] - 5:,} more rows")
35
36def quick_stats(df, numeric_only=True):
37    """Quick statistical overview."""
38    
39    if numeric_only:
40        numeric_cols = df.select_dtypes(include=[np.number])
41        if len(numeric_cols.columns) > 0:
42            print("📊 Numeric Column Statistics:")
43            display(numeric_cols.describe())
44    
45    print("\n📋 Column Information:")
46    info_df = pd.DataFrame({
47        'Column': df.columns,
48        'Type': df.dtypes,
49        'Non-Null': df.count(),
50        'Null Count': df.isnull().sum(),
51        'Null %': (df.isnull().sum() / len(df) * 100).round(2)
52    })
53    display(info_df)
54
55# Apply setup
56setup_pandas_display()
57
58# Example usage
59display_with_info(df_enhanced, "Enhanced Sales Dataset")
60quick_stats(df_enhanced)

 1# Debugging patterns for pandas in Jupyter
 2
 3# Pattern 1: Step-by-step data transformation
 4def debug_transformation(df):
 5    """Debug complex data transformations step by step."""
 6    
 7    print("🐛 DEBUGGING DATA TRANSFORMATION")
 8    print("=" * 40)
 9    
10    # Start with the data
11    print("Step 1: Original data")
12    print(f"Shape: {df.shape}")
13    display(df.head(3))
14    
15    # Step 2: Apply first transformation
16    print("\nStep 2: Filter for recent data")
17    recent_cutoff = df['date'].max() - pd.Timedelta(days=90)
18    df_recent = df[df['date'] >= recent_cutoff]
19    print(f"Shape after date filter: {df_recent.shape}")
20    print(f"Removed {len(df) - len(df_recent):,} rows")
21    
22    # Step 3: Group and aggregate
23    print("\nStep 3: Group by product and region")
24    grouped = df_recent.groupby(['product', 'region'])['sales'].agg(['sum', 'count', 'mean'])
25    print(f"Grouped data shape: {grouped.shape}")
26    display(grouped.head())
27    
28    # Step 4: Apply business logic
29    print("\nStep 4: Apply business rules")
30    # Only products with more than 10 sales
31    filtered_groups = grouped[grouped['count'] >= 10]
32    print(f"After count filter: {filtered_groups.shape}")
33    
34    # Only above-average performers
35    avg_sales = filtered_groups['sum'].median()
36    final_result = filtered_groups[filtered_groups['sum'] >= avg_sales]
37    print(f"Final result shape: {final_result.shape}")
38    print(f"Average sales threshold: ${avg_sales:,.0f}")
39    
40    display(final_result.head())
41    
42    return final_result
43
44# Pattern 2: Data validation checkpoints
45def validate_data_pipeline(df, checkpoints=True):
46    """Data pipeline with validation checkpoints."""
47    
48    original_count = len(df)
49    
50    # Checkpoint 1: Remove nulls
51    df_clean = df.dropna(subset=['sales', 'product', 'region'])
52    if checkpoints:
53        print(f"✓ Checkpoint 1: Removed {original_count - len(df_clean)} null records")
54    
55    # Checkpoint 2: Remove invalid sales
56    df_valid = df_clean[df_clean['sales'] > 0]
57    if checkpoints:
58        print(f"✓ Checkpoint 2: Removed {len(df_clean) - len(df_valid)} zero/negative sales")
59    
60    # Checkpoint 3: Remove outliers
61    q99 = df_valid['sales'].quantile(0.99)
62    df_no_outliers = df_valid[df_valid['sales'] <= q99]
63    if checkpoints:
64        print(f"✓ Checkpoint 3: Removed {len(df_valid) - len(df_no_outliers)} outliers (sales > ${q99:.0f})")
65    
66    # Final validation
67    if checkpoints:
68        print(f"\n📊 Pipeline Summary:")
69        print(f"   Started with: {original_count:,} records")
70        print(f"   Ended with: {len(df_no_outliers):,} records")
71        print(f"   Data retention: {len(df_no_outliers)/original_count:.1%}")
72    
73    return df_no_outliers
74
75# Usage
76debug_result = debug_transformation(df)
77clean_data = validate_data_pipeline(df)