Clustering MLB Players into Performance Archetypes Using SVM and Feature Cutoffs¶

data = pd.read_csv("/content/fangraphs-leaderboards.csv", encoding='latin1')
data.head()

# Check for duplicate column names
duplicate_column_names = data.columns[data.columns.duplicated()].tolist()

# Display duplicate column names
print("Duplicate Column Names:", duplicate_column_names)

# Transpose the DataFrame and drop duplicates based on identical column values
duplicate_columns = data.T.duplicated()

# List the columns that are duplicated
repeated_columns = data.columns[duplicate_columns]

# Display the repeated columns
print("Repeated Columns:", repeated_columns)

# Transpose the DataFrame and check for duplicated columns based on identical content
duplicate_columns = data.T.duplicated()

# Get the names of the duplicated columns
repeated_columns = data.columns[duplicate_columns]

# Drop the duplicated columns from the DataFrame
data = data.drop(columns=repeated_columns)

# Display the remaining columns to verify that duplicates are removed
print(f"Deleted {len(repeated_columns)} duplicated columns.")

# Display a list of all column names
column_names = data.columns.tolist()

# Print the list of column names
print("List of columns in the DataFrame:")
print(column_names)

# Rename the column from 'Season' to 'Season'
data.rename(columns={'Season': 'Season'}, inplace=True)

# Verify the column renaming
print(data.columns)

# Check for missing values
data.isnull().sum()

# drop columns with more than 20% missing values
threshold = len(data) * 0.80

# Drop columns with more than 20% missing values
data = data.dropna(axis=1, thresh=threshold)

# Recalculate the percentage of missing values per column
missing_percentages = (data.isnull().sum() / len(data)) * 100

# Sort and display the top 15 columns with the highest percentage of missing values
top_15_null_columns = missing_percentages.sort_values(ascending=False).head(15)

# Display
print(top_15_null_columns)

# Select only numeric columns (int and float) for skewness and imputation
numeric_columns = data.select_dtypes(include=['float64', 'int64'])

# Calculate skewness for numeric columns only
skewness = numeric_columns.skew()

# Define a function to impute based on skewness
def impute_based_on_skewness(df, skewness):
    for column in df.columns:
        if df[column].isnull().sum() > 0:
            if abs(skewness[column]) > 0.5:
                df[column] = df[column].fillna(df[column].median())
            else:  #
                df[column] = df[column].fillna(df[column].mean())

# Apply the function to impute missing values for the numeric data
impute_based_on_skewness(numeric_columns, skewness)

# Replace the numeric columns in the original dataframe with the imputed ones
data[numeric_columns.columns] = numeric_columns

# Check the result to see if missing values are imputed
print(data.isnull().sum())

# Display only the columns with missing values
missing_columns = data.columns[data.isnull().any()]  # Get columns with missing values
missing_data = data[missing_columns].isnull().sum()  # Count missing values in those columns

# Display the columns with their missing value count
print(missing_data)

# Check the data types of the problematic columns
print(data[['Age Rng', 'Bats']].dtypes)
# Display the first few rows of the problematic columns
print(data[['Age Rng', 'Bats']].head(10))

# Check for missing values in the dataset
missing_values = data.isnull().sum()

# Filter and display only columns with missing values
columns_with_missing_values = missing_values[missing_values > 0]

# Display the columns with missing values
print(columns_with_missing_values)

# Calculate skewness for Min Age and Max Age
min_age_skewness = data['Min Age'].skew()
max_age_skewness = data['Max Age'].skew()

print(f"Skewness of 'Min Age': {min_age_skewness:.2f}")
print(f"Skewness of 'Max Age': {max_age_skewness:.2f}")

# Plot histograms to visualize the distribution
plt.figure(figsize=(10, 5))

plt.subplot(1, 2, 1)
data['Min Age'].hist(bins=20)
plt.title('Min Age Distribution')

plt.subplot(1, 2, 2)
data['Max Age'].hist(bins=20)
plt.title('Max Age Distribution')

plt.tight_layout()
plt.show()

# Impute missing values in Min Age and Max Age using the mean
data['Min Age'].fillna(data['Min Age'].mean(), inplace=True)
data['Max Age'].fillna(data['Max Age'].mean(), inplace=True)

# Verify that there are no more missing values
print(data[['Min Age', 'Max Age']].isnull().sum())

# Check the range of values in numeric columns
range_check = data.select_dtypes(include=['float64', 'int64']).agg(['min', 'max'])

# Display range check
print(range_check)

# Step 1: Retain 'PlayerId', 'MLBAMID', 'Season', and 'Name' for later analysis (exclude them from PCA)
columns_to_exclude = ['PlayerId', 'MLBAMID', 'Season', 'Name']
features_for_pca = data.drop(columns=columns_to_exclude)

# Step 2: Select only numeric columns for PCA
numeric_features = features_for_pca.select_dtypes(include=['float64', 'int64'])

# Step 3: Standardize the features before applying PCA
scaler = StandardScaler()
scaled_features = scaler.fit_transform(numeric_features)

# Step 4: Apply PCA (retain 95% of variance)
pca = PCA(n_components=0.95)
pca_features = pca.fit_transform(scaled_features)

# Display how many components were retained after PCA
print(f"PCA reduced the dataset to {pca_features.shape[1]} components.")

# Step 5: Visualize the explained variance by each principal component
explained_variance = pca.explained_variance_ratio_

plt.figure(figsize=(8, 5))
plt.plot(range(1, len(explained_variance) + 1), explained_variance.cumsum(), marker='o', linestyle='--')
plt.title('Explained Variance by Principal Components')
plt.xlabel('Number of Components')
plt.ylabel('Cumulative Explained Variance')
plt.grid(True)
plt.show()

# Step 1: Exclude non-performance columns AND 'Age', 'Min Age', 'Max Age' for clustering
columns_to_exclude = ['PlayerId', 'MLBAMID', 'Season', 'Name', 'Age', 'Min Age', 'Max Age', 'Pitches','Strikes','Balls', 'AB', 'PA']
features_for_kmeans = data.drop(columns=columns_to_exclude)

# Step 2: Initialize KMeans model (with 3 clusters, adjust based on your needs)
kmeans = KMeans(n_clusters=3, random_state=42)

# Step 3: Fit the KMeans model on PCA-transformed features
kmeans.fit(pca_features)

# Step 4: Get cluster labels for each player (these labels will be used as the target in SVM)
cluster_labels = kmeans.labels_

# Step 5: Retain 'PlayerId', 'MLBAMID', 'Season', and 'Name' for post-analysis (IDs and seasons)
ids_and_season = data[['PlayerId', 'MLBAMID', 'Season', 'Name']]

# Step 6: Combine the IDs, seasons, and cluster labels to review
final_result = pd.concat([ids_and_season.reset_index(drop=True), pd.DataFrame(cluster_labels, columns=['Cluster'])], axis=1)

# Step 7: View the final result to see which players belong to each cluster
print(final_result.head())

# Get the unique clusters and their counts
cluster_counts = pd.Series(cluster_labels).value_counts()

# Display the number of players in each cluster
print(cluster_counts)

# Display the total number of unique clusters
print(f"Total number of clusters: {len(cluster_counts)}")

# Step 1: Calculate inertia (within-cluster sum of squares) for different cluster sizes
inertia_values = []
cluster_range = range(2, 11)  # Test cluster sizes from 2 to 10

for n_clusters in cluster_range:
    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
    kmeans.fit(pca_features)
    inertia_values.append(kmeans.inertia_)

# Step 2: Plot the inertia values to find the "elbow"
plt.figure(figsize=(8, 5))
plt.plot(cluster_range, inertia_values, marker='o', linestyle='--')
plt.title('Elbow Method for Optimal Clusters')
plt.xlabel('Number of Clusters')
plt.ylabel('Inertia (Within-Cluster Sum of Squares)')
plt.grid(True)
plt.show()

# Step 1: Initialize KMeans with 4 clusters
optimal_n_clusters = 4
kmeans = KMeans(n_clusters=optimal_n_clusters, random_state=42)

# Step 2: Fit the KMeans model on PCA-transformed features (excluding age-related columns)
kmeans.fit(pca_features)

# Step 3: Get the new cluster labels
cluster_labels = kmeans.labels_

# Step 4: Retain 'PlayerId', 'MLBAMID', 'Season', and 'Name' for post-analysis (IDs and seasons)
ids_and_season = data[['PlayerId', 'MLBAMID', 'Season', 'Name']]

# Step 5: Combine the IDs, seasons, and cluster labels
final_result = pd.concat([ids_and_season.reset_index(drop=True), pd.DataFrame(cluster_labels, columns=['Cluster'])], axis=1)

# Step 6: View the final result
print(final_result.head())

# Get the unique clusters and their counts
cluster_counts = pd.Series(cluster_labels).value_counts()

# Display the number of players in each cluster
print(cluster_counts)

# Display the total number of unique clusters
print(f"Total number of clusters: {len(cluster_counts)}")

# Step 1: Exclude non-performance columns (PlayerId, MLBAMID, Season, Name) for SVM
features_for_svm = data.drop(columns=columns_to_exclude)

# Step 2: Select only numeric columns for SVM
numeric_features = features_for_svm.select_dtypes(include=['float64', 'int64'])

# Step 3: Split the PCA-transformed numeric data into training and test sets
X_train, X_test, y_train, y_test = train_test_split(pca_features, cluster_labels, test_size=0.3, random_state=42)

# Step 4: Initialize and train the SVM model
svm_model = SVC(kernel='linear')
svm_model.fit(X_train, y_train)

# Step 5: Evaluate the SVM model
print(f"Training Accuracy: {svm_model.score(X_train, y_train)}")
print(f"Test Accuracy: {svm_model.score(X_test, y_test)}")

# Step 6: Make predictions on the test set
predicted_labels = svm_model.predict(X_test)

# Step 7: Combine the IDs, seasons, and predicted labels for final analysis
final_result_svm = pd.concat([ids_and_season.reset_index(drop=True), pd.DataFrame(predicted_labels, columns=['Predicted Cluster'])], axis=1)

# Step 8: View the final result (showing predicted clusters)
print(final_result_svm.head())

# Step 1: Exclude non-performance columns (PlayerId, MLBAMID, Season, Name) for SVM
features_for_svm = data.drop(columns=columns_to_exclude)

# Step 2: Select only numeric columns for SVM
numeric_features = features_for_svm.select_dtypes(include=['float64', 'int64'])

# Step 3: Perform cross-validation using the PCA-transformed features and cluster labels as target
svm_model = SVC(kernel='linear')

# Perform 5-fold cross-validation (using PCA-transformed features and cluster labels)
cv_scores = cross_val_score(svm_model, pca_features, cluster_labels, cv=5)

# Step 4: Display the cross-validation scores and the mean score (rounded to 3 decimal places)
cv_scores_rounded = [round(score, 3) for score in cv_scores]
mean_cv_score_rounded = round(cv_scores.mean(), 3)

# Step 5: Print the rounded scores
print(f"Cross-validation scores for each fold: {cv_scores_rounded}")
print(f"Mean cross-validation accuracy: {mean_cv_score_rounded}")

# Impute NaNs with the median value of each column
numeric_columns = numeric_columns.fillna(numeric_columns.median())

# Check again for any remaining NaNs
print(numeric_columns.isnull().sum().sum())

# Apply t-SNE for 2D visualization of clusters
tsne = TSNE(n_components=2, random_state=42)
tsne_features = tsne.fit_transform(numeric_columns)

# Plot the t-SNE results
plt.figure(figsize=(8, 6))
plt.scatter(tsne_features[:, 0], tsne_features[:, 1], c=cluster_labels, cmap='viridis', s=50)
plt.title('t-SNE Visualization of Clusters')
plt.xlabel('t-SNE Component 1')
plt.ylabel('t-SNE Component 2')
plt.colorbar(label='Cluster')
plt.grid(True)

# Save the figure as a PNG file before displaying
plt.savefig("tsne_plot.png", format='png')

# Display the plot in the notebook
plt.show()

# Step 1: Add the predicted cluster labels from the SVM model to the original dataset
final_result_svm = pd.concat([ids_and_season.reset_index(drop=True), pd.DataFrame(predicted_labels, columns=['Predicted Cluster'])], axis=1)

# Step 2: Combine the predicted clusters with the original data
data_with_predicted_clusters = pd.concat([data, pd.DataFrame(predicted_labels, columns=['Predicted Cluster'])], axis=1)

# Step 3: Exclude non-performance columns like 'PlayerId', 'MLBAMID', 'Season', and 'Name'
performance_data = data_with_predicted_clusters.drop(columns=columns_to_exclude)

# Step 4: Select only numeric performance-related columns for analysis
numeric_columns = performance_data.select_dtypes(include=['float64', 'int64'])

# Step 5: Calculate the mean values of the numeric performance features for each predicted cluster
cluster_means_svm = numeric_columns.groupby('Predicted Cluster').mean()

# Step 6: Display the mean values of features for each predicted cluster
print(cluster_means_svm)

# Calculate the mean for each cluster across all numeric features
cluster_means = numeric_columns.groupby(cluster_labels).mean()

# Calculate the variance for each cluster across all numeric features
cluster_variance = numeric_columns.groupby(cluster_labels).var()

# Display the top metrics with the highest variance across clusters
top_variance_metrics = cluster_variance.mean().sort_values(ascending=False).head(10)
print("Top 10 metrics with highest variance across clusters:")
print(top_variance_metrics)

# Display the means for top variance metrics
print("Cluster means for top variance metrics:")
print(cluster_means[top_variance_metrics.index])

# Visualize the means of top variance features using a bar chart
high_variance_features = top_variance_metrics.index  # Features with high variance

# Calculate means for these features by cluster
cluster_means_top_variance = cluster_means[high_variance_features]

# Plot the top variance features by cluster
cluster_means_top_variance.T.plot(kind='bar', figsize=(12, 8))
plt.title("Top-Variance Performance Metrics by Predicted Cluster")
plt.ylabel("Mean Value")
plt.xlabel("Performance Metrics")
plt.xticks(rotation=45)
plt.legend(title="Cluster")
plt.grid(True)
plt.tight_layout()
plt.show()

# Impute NaNs with the median value of each column
numeric_columns = numeric_columns.fillna(numeric_columns.median())

# Check again for any remaining NaNs
print(numeric_columns.isnull().sum().sum())

# Compare key metrics for Cluster 0 and Cluster 3
key_metrics = ['HR', 'ISO', 'SB', 'K%', 'BB%']

# cluster labels
cluster_labels_series = pd.Series(cluster_labels)

# Filter the data for Cluster 0 and Cluster 3
cluster_0_3_data = numeric_columns[cluster_labels_series.isin([0, 3])]

# Compare mean values of key metrics for both clusters
cluster_0_3_means = cluster_0_3_data.groupby(cluster_labels_series).mean()[key_metrics]

# Display the results
print(cluster_0_3_means)

# Assigning descriptive names to the clusters
cluster_names = {
    0: "Speedy Contact Hitters",
    1: "Defensive Specialists",
    2: "Power-Speed Hybrids",
    3: "Power Hitters"
}


# Function to look up a player and return their cluster information
def get_player_cluster(player_name):
    # Search for the player in the DataFrame
    player_info = final_result[final_result['Name'].str.contains(player_name, case=False, na=False)]

    # If player is found
    if not player_info.empty:
        cluster_label = player_info.iloc[0]['Cluster']  # Get the cluster label
        cluster_name = cluster_names[cluster_label]  # Get the cluster name from the dictionary
        return f"{player_name} belongs to '{cluster_name}' (Cluster {cluster_label})"
    else:
        return f"Player {player_name} not found in the dataset."

# Example usage:
print(get_player_cluster("Randy Arozarena"))

print(get_player_cluster("Francisco Lindor"))

!cp "/content/drive/MyDrive/Colab Notebooks/silverstein_svm.ipynb" ./
!jupyter nbconvert --to html "silverstein_svm.ipynb"