# 数据处理库
import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')

# 可视化库 - pyecharts (使用离线模式)
from pyecharts import options as opts
from pyecharts.charts import Bar, Pie, Line, Scatter, Radar, HeatMap, Grid, Tab
from pyecharts.globals import ThemeType, CurrentConfig
from pyecharts.commons.utils import JsCode

# 机器学习库
from sklearn.cluster import KMeans  # K-Means聚类算法
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler  # 数据标准化
from sklearn.metrics import silhouette_score, davies_bouldin_score, calinski_harabasz_score  # 聚类评估指标
from sklearn.decomposition import PCA  # 主成分分析降维
from sklearn.manifold import TSNE  # t-SNE降维（更适合可视化）

print("所有库导入成功！")

# 读取数据
df = pd.read_csv('user_personalized_features.csv', index_col=['Unnamed: 0'])

# 数据预览
print("="*70)
print(" 数据预览（前5行）")
print("="*70)
display(df.head())

# 数据维度
print("\n" + "="*70)
print(f"数据集维度：{df.shape[0]} 行 × {df.shape[1]} 列")
print("="*70)

# 数据质量检查
print("\n" + "="*70)
print(" 数据质量检查")
print("="*70)
print(f"✓ 重复值数量：{df.duplicated().sum()} 条")
print(f"✓ 缺失值总数：{df.isna().sum().sum()} 个")
if df.isna().sum().sum() > 0:
    print("\n各字段缺失值详情：")
    print(df.isna().sum()[df.isna().sum() > 0])
    
# 数据类型信息
print("\n" + "="*70)
print(" 数据类型信息")
print("="*70)
display(df.info())

# 描述性统计
print("\n" + "="*70)
print(" 数值型字段描述性统计")
print("="*70)
display(df.describe().round(2))

# 分类字段统计
print("\n" + "="*70)
print("分类字段唯一值统计")
print("="*70)
categorical_cols = df.select_dtypes(include=['object', 'bool']).columns
for col in categorical_cols:
    if col != 'User_ID':
        print(f"\n{col}: {df[col].nunique()} 个唯一值")
        print(f"  {df[col].unique()}")

======================================================================
 数据预览（前5行）
======================================================================

======================================================================
 数据集维度：1000 行 × 14 列
======================================================================

======================================================================
 数据质量检查
======================================================================
✓ 重复值数量：0 条
✓ 缺失值总数：0 个

======================================================================
📋 数据类型信息
======================================================================
<class 'pandas.core.frame.DataFrame'>
Index: 1000 entries, 0 to 999
Data columns (total 14 columns):
 #   Column                       Non-Null Count  Dtype 
---  ------                       --------------  ----- 
 0   User_ID                      1000 non-null   object
 1   Age                          1000 non-null   int64 
 2   Gender                       1000 non-null   object
 3   Location                     1000 non-null   object
 4   Income                       1000 non-null   int64 
 5   Interests                    1000 non-null   object
 6   Last_Login_Days_Ago          1000 non-null   int64 
 7   Purchase_Frequency           1000 non-null   int64 
 8   Average_Order_Value          1000 non-null   int64 
 9   Total_Spending               1000 non-null   int64 
 10  Product_Category_Preference  1000 non-null   object
 11  Time_Spent_on_Site_Minutes   1000 non-null   int64 
 12  Pages_Viewed                 1000 non-null   int64 
 13  Newsletter_Subscription      1000 non-null   bool  
dtypes: bool(1), int64(8), object(5)
memory usage: 110.4+ KB

======================================================================
 数值型字段描述性统计
======================================================================

# 提取所有分类字段的唯一值
print("="*70)
print("📋 分类字段详细信息")
print("="*70)

categorical_features = {}
for col in df.columns:
    if df[col].dtype == 'object' and col != 'User_ID':
        unique_values = df[col].unique()
        count_values = df[col].value_counts()
        categorical_features[col] = {
            'unique_count': len(unique_values),
            'values': unique_values,
            'distribution': count_values
        }
        print(f"\n【{col}】")
        print(f"  唯一值数量: {len(unique_values)}")
        print(f"  唯一值: {unique_values}")
        print(f"  分布情况:")
        for val, cnt in count_values.items():
            print(f"    - {val}: {cnt} ({cnt/len(df)*100:.1f}%)")

======================================================================
📋 分类字段详细信息
======================================================================

【Gender】
  唯一值数量: 2
  唯一值: ['Male' 'Female']
  分布情况:
    - Male: 526 (52.6%)
    - Female: 474 (47.4%)

【Location】
  唯一值数量: 3
  唯一值: ['Suburban' 'Rural' 'Urban']
  分布情况:
    - Suburban: 349 (34.9%)
    - Urban: 344 (34.4%)
    - Rural: 307 (30.7%)

【Interests】
  唯一值数量: 5
  唯一值: ['Sports' 'Technology' 'Fashion' 'Travel' 'Food']
  分布情况:
    - Sports: 213 (21.3%)
    - Fashion: 209 (20.9%)
    - Travel: 196 (19.6%)
    - Food: 196 (19.6%)
    - Technology: 186 (18.6%)

【Product_Category_Preference】
  唯一值数量: 5
  唯一值: ['Books' 'Electronics' 'Apparel' 'Health & Beauty' 'Home & Kitchen']
  分布情况:
    - Apparel: 218 (21.8%)
    - Electronics: 209 (20.9%)
    - Books: 198 (19.8%)
    - Home & Kitchen: 189 (18.9%)
    - Health & Beauty: 186 (18.6%)

# 1. 性别分布 - 饼图
gender_count = df['Gender'].value_counts()
pie_gender = (
    Pie(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="400px"))
    .add(
        series_name="性别分布",
        data_pair=[(k, int(v)) for k, v in gender_count.items()],
        radius=["30%", "60%"],
        label_opts=opts.LabelOpts(formatter="{b}: {c}人 ({d}%)"),
    )
    .set_global_opts(
        title_opts=opts.TitleOpts(title="用户性别分布", subtitle=f"总用户数: {len(df)}", pos_left="center"),
        legend_opts=opts.LegendOpts(orient="vertical", pos_right="5%", pos_top="15%"),
    )
    .set_series_opts(
        tooltip_opts=opts.TooltipOpts(trigger="item", formatter="{b}: {c}人 ({d}%)")
    )
)
pie_gender.render_notebook()

# 2. 地区分布 - 柱状图
location_count = df['Location'].value_counts().sort_values(ascending=False)
bar_location = (
    Bar(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="400px"))
    .add_xaxis(location_count.index.tolist())
    .add_yaxis(
        "用户数量",
        location_count.values.tolist(),
        label_opts=opts.LabelOpts(is_show=True, position="top"),
    )
    .set_global_opts(
        title_opts=opts.TitleOpts(title="用户地区分布", subtitle="不同地区用户数量统计", pos_left="center"),
        xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=0)),
        yaxis_opts=opts.AxisOpts(name="用户数量"),
        toolbox_opts=opts.ToolboxOpts(),
    )
)
bar_location.render_notebook()

# 3. 兴趣分布 - 柱状图
interests_count = df['Interests'].value_counts().sort_values(ascending=False)
bar_interests = (
    Bar(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="400px"))
    .add_xaxis(interests_count.index.tolist())
    .add_yaxis(
        "用户数量",
        interests_count.values.tolist(),
        label_opts=opts.LabelOpts(is_show=True, position="top"),
    )
    .set_global_opts(
        title_opts=opts.TitleOpts(title="用户兴趣分布", subtitle="不同兴趣爱好用户数量", pos_left="center"),
        xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=-15)),
        yaxis_opts=opts.AxisOpts(name="用户数量"),
    )
)
bar_interests.render_notebook()

# 4. 产品类别偏好 - 柱状图
product_count = df['Product_Category_Preference'].value_counts().sort_values(ascending=False)
bar_product = (
    Bar(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="400px"))
    .add_xaxis(product_count.index.tolist())
    .add_yaxis(
        "用户数量",
        product_count.values.tolist(),
        label_opts=opts.LabelOpts(is_show=True, position="top"),
    )
    .set_global_opts(
        title_opts=opts.TitleOpts(title="产品类别偏好分布", subtitle="用户偏好的产品类别统计", pos_left="center"),
        xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=-15)),
        yaxis_opts=opts.AxisOpts(name="用户数量"),
    )
)
bar_product.render_notebook()

# 5. 新闻订阅分布 - 饼图
newsletter_count = df['Newsletter_Subscription'].value_counts()
newsletter_data = [["已订阅" if k else "未订阅", int(v)] for k, v in newsletter_count.items()]
pie_newsletter = (
    Pie(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="400px"))
    .add(
        series_name="订阅状态",
        data_pair=newsletter_data,
        radius=["30%", "60%"],
        label_opts=opts.LabelOpts(formatter="{b}: {c}人 ({d}%)"),
    )
    .set_global_opts(
        title_opts=opts.TitleOpts(title="营销邮件订阅分布", subtitle=f"总用户数: {len(df)}", pos_left="center"),
        legend_opts=opts.LegendOpts(orient="vertical", pos_right="5%", pos_top="15%"),
    )
    .set_colors(["#5470c6", "#91cc75"])
)
pie_newsletter.render_notebook()

# 创建分析数据集副本
act_df = df.copy()

# 年龄分段
act_df['Age_Group'] = pd.cut(
    act_df['Age'],
    bins=[18, 25, 40, 60, np.inf],
    labels=['18-24岁青少年', '25-39岁青年', '40-59岁中年', '60岁以上老年']
)

# 转换时间单位（分钟→小时）便于可视化
act_df['Time_Spent_on_Site_Hours'] = (act_df['Time_Spent_on_Site_Minutes'] / 60).round(2)

print(f"\n年龄组分布：")
print(act_df['Age_Group'].value_counts().sort_index())

年龄组分布：
Age_Group
18-24岁青少年    152
25-39岁青年     295
40-59岁中年     440
60岁以上老年       88
Name: count, dtype: int64

# 计算性别维度的平均活跃度指标
gender_activity = act_df.groupby('Gender')[
    ['Purchase_Frequency', 'Last_Login_Days_Ago', 'Time_Spent_on_Site_Hours', 'Pages_Viewed']
].mean().round(2)

print("性别维度活跃度指标：")
display(gender_activity)

# 柱状图对比
metrics = ['Purchase_Frequency', 'Last_Login_Days_Ago', 'Time_Spent_on_Site_Hours', 'Pages_Viewed']
metric_names = ['购买频率', '最近登录天数', '停留时长(小时)', '浏览页数']

bar_gender = (
    Bar(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="400px"))
    .add_xaxis(metric_names)
)

for gender in gender_activity.index:
    values = [gender_activity.loc[gender, m] for m in metrics]
    bar_gender.add_yaxis(
        gender,
        [round(v, 2) for v in values],
        label_opts=opts.LabelOpts(is_show=True, position="top"),
    )

bar_gender.set_global_opts(
    title_opts=opts.TitleOpts(title="性别维度用户活跃度对比", subtitle="各指标平均值", pos_left="center"),
    yaxis_opts=opts.AxisOpts(name="平均值"),
    xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=-15)),
    toolbox_opts=opts.ToolboxOpts(),
    legend_opts=opts.LegendOpts(pos_top="8%"),
)
bar_gender.render_notebook()

# 计算地区维度的平均活跃度指标
location_activity = act_df.groupby('Location')[
    ['Purchase_Frequency', 'Last_Login_Days_Ago', 'Time_Spent_on_Site_Hours', 'Pages_Viewed']
].mean().round(2)

print("地区维度活跃度指标：")
display(location_activity)

# 柱状图对比
bar_location_activity = (
    Bar(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="400px"))
    .add_xaxis(metric_names)
)

for location in location_activity.index:
    values = [location_activity.loc[location, m] for m in metrics]
    bar_location_activity.add_yaxis(
        location,
        [round(v, 2) for v in values],
        label_opts=opts.LabelOpts(is_show=True, position="top", font_size=10),
    )

bar_location_activity.set_global_opts(
    title_opts=opts.TitleOpts(title="地区维度用户活跃度对比", subtitle="各指标平均值", pos_left="center"),
    yaxis_opts=opts.AxisOpts(name="平均值"),
    xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=-15)),
    toolbox_opts=opts.ToolboxOpts(),
    legend_opts=opts.LegendOpts(pos_top="8%"),
)
bar_location_activity.render_notebook()

# 计算年龄维度的平均活跃度指标
age_activity = act_df.groupby('Age_Group')[
    ['Purchase_Frequency', 'Last_Login_Days_Ago', 'Time_Spent_on_Site_Hours', 'Pages_Viewed']
].mean().round(2)

print("年龄维度活跃度指标：")
display(age_activity)

# 柱状图对比
bar_age_activity = (
    Bar(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="400px"))
    .add_xaxis(metric_names)
)

for age_group in age_activity.index:
    values = [age_activity.loc[age_group, m] for m in metrics]
    bar_age_activity.add_yaxis(
        age_group,
        [round(v, 2) for v in values],
        label_opts=opts.LabelOpts(is_show=True, position="top", font_size=9),
    )

bar_age_activity.set_global_opts(
    title_opts=opts.TitleOpts(title="年龄维度用户活跃度对比", subtitle="各指标平均值", pos_left="center"),
    yaxis_opts=opts.AxisOpts(name="平均值"),
    xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=-15)),
    toolbox_opts=opts.ToolboxOpts(),
    legend_opts=opts.LegendOpts(pos_top="8%"),
)
bar_age_activity.render_notebook()

# 步骤1：提取RFM原始数据
rfm_df = df[['User_ID', 'Last_Login_Days_Ago', 'Purchase_Frequency', 'Total_Spending']].copy()

# 重命名列为标准的RFM命名
rfm_df = rfm_df.rename(columns={
    'Last_Login_Days_Ago': 'R',  # Recency - 最近登录天数
    'Purchase_Frequency': 'F',    # Frequency - 购买频率  
    'Total_Spending': 'M'         # Monetary - 总消费金额
})

print("="*70)
print(" RFM原始数据预览")
print("="*70)
display(rfm_df.head(10))

print("\n" + "="*70)
print(" RFM指标描述性统计")
print("="*70)
display(rfm_df[['R', 'F', 'M']].describe().round(2))

# 步骤2：计算RFM分数（1-5分）

# R分数：天数越小，分数越高（使用负值反转）
rfm_df['R_Score'] = pd.qcut(
    -rfm_df['R'],           # 使用负值反转，使得天数小的用户得高分
    q=5,                    # 分为5个等级
    labels=False,           # 返回数值而非区间标签
    duplicates='drop'       # 处理重复边界值
) + 1                       # 从1开始计分（而非0）

# F分数：频率越高，分数越高
rfm_df['F_Score'] = pd.qcut(
    rfm_df['F'],
    q=5,
    labels=False,
    duplicates='drop'
) + 1

# M分数：消费金额越高，分数越高
rfm_df['M_Score'] = pd.qcut(
    rfm_df['M'],
    q=5,
    labels=False,
    duplicates='drop'
) + 1

print("="*70)
print(" RFM分数计算完成")
print("="*70)
display(rfm_df.head(10))

======================================================================
 RFM分数计算完成
======================================================================

======================================================================
 RFM分数分布统计
======================================================================

R分数分布:
R_Score
1    210
2    220
3    193
4    201
5    176
Name: count, dtype: int64

F分数分布:
F_Score
1    280
2    192
3    227
4    192
5    109
Name: count, dtype: int64

M分数分布:
M_Score
1    200
2    200
3    200
4    201
5    199
Name: count, dtype: int64

# 步骤3：设计RFM标签分类函数

def rfm_label(row):
    """根据RFM分数为用户打标签"""
    R, F, M = row['R_Score'], row['F_Score'], row['M_Score']
    
    # 高消费用户（M >= 4）- 重要客户
    if M >= 4:
        if R >= 4 and F >= 4:
            return '重要价值客户'     # VIP
        elif R < 4 and F >= 4:
            return '重要保持客户'     # 需唤回
        elif R >= 4 and F < 4:
            return '重要发展客户'     # 潜力股
        else:  # R < 4 and F < 4
            return '重要挽留客户'     # 流失风险
    
    # 低消费用户（M < 4）- 一般客户
    else:
        if R >= 4 and F >= 4:
            return '一般价值客户'     # 可提升客单价
        elif R < 4 and F >= 4:
            return '一般保持客户'     # 维护忠诚度
        elif R >= 4 and F < 4:
            return '一般发展客户'     # 新用户
        else:  # R < 4 and F < 4
            return '一般挽留客户'     # 已流失

# 应用标签函数
rfm_df['RFM_Label'] = rfm_df.apply(rfm_label, axis=1)

# 用户标签分类
label_counts = rfm_df['RFM_Label'].value_counts()
for label, count in label_counts.items():
    percentage = count / len(rfm_df) * 100
    print(f"{label}: {count} 人 ({percentage:.1f}%)")

一般挽留客户: 257 人 (25.7%)
重要挽留客户: 171 人 (17.1%)
一般发展客户: 169 人 (16.9%)
一般保持客户: 111 人 (11.1%)
重要发展客户: 102 人 (10.2%)
重要保持客户: 84 人 (8.4%)
一般价值客户: 63 人 (6.3%)
重要价值客户: 43 人 (4.3%)

# 1. 饼图：用户分类占比
label_counts = rfm_df['RFM_Label'].value_counts()
pie_data = [(label, int(count)) for label, count in label_counts.items()]

# 定义颜色方案：重要客户用暖色，一般客户用冷色
color_map = {
    '重要价值客户': '#ee5a6f',
    '重要保持客户': '#f78ca0',
    '重要发展客户': '#fac858',
    '重要挽留客户': '#ff9800',
    '一般价值客户': '#5470c6',
    '一般保持客户': '#73c0de',
    '一般发展客户': '#91cc75',
    '一般挽留客户': '#9a60b4'
}

pie_rfm = (
    Pie(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="500px"))
    .add(
        series_name="用户数量",
        data_pair=pie_data,
        radius=["35%", "65%"],
        label_opts=opts.LabelOpts(
            formatter="{b}\n{c}人 ({d}%)",
            font_size=11,
        ),
        itemstyle_opts=opts.ItemStyleOpts(
            border_color="#fff",
            border_width=2
        ),
    )
    .set_global_opts(
        title_opts=opts.TitleOpts(
            title="RFM用户价值分布",
            subtitle=f"总用户数: {len(rfm_df)}",
            pos_left="center",
            title_textstyle_opts=opts.TextStyleOpts(font_size=20),
        ),
        legend_opts=opts.LegendOpts(
            orient="vertical",
            pos_right="3%",
            pos_top="15%",
            item_width=15,
            item_height=15,
        ),
        toolbox_opts=opts.ToolboxOpts(),
    )
    .set_colors([color_map.get(item[0], '#5470c6') for item in pie_data])
)
pie_rfm.render_notebook()

# 2. 柱状图：用户分类数量
bar_rfm = (
    Bar(init_opts=opts.InitOpts(theme=ThemeType.WALDEN, width="900px", height="450px"))
    .add_xaxis(list(label_counts.index))
    .add_yaxis(
        "用户数量",
        list(label_counts.values),
        label_opts=opts.LabelOpts(is_show=True, position="top"),
    )
    .set_global_opts(
        title_opts=opts.TitleOpts(title="RFM用户分类数量统计", pos_left="center"),
        xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=-30, font_size=10)),
        yaxis_opts=opts.AxisOpts(name="用户数量"),
        toolbox_opts=opts.ToolboxOpts(),
    )
)
bar_rfm.render_notebook()

# 整合：年龄、收入、RFM指标、行为特征

# 步骤1：构建特征集
cluster_features = df[[
    # 用户属性
    'Age',
    'Income',
    # RFM核心指标
    'Last_Login_Days_Ago',      # R - 最近登录
    'Purchase_Frequency',        # F - 购买频率
    'Total_Spending',            # M - 总消费
    # 行为特征
    'Average_Order_Value',       # 平均订单价值
    'Time_Spent_on_Site_Minutes',  # 停留时长
    'Pages_Viewed',              # 浏览页数
]].copy()

# 步骤2：特征工程 - 创建衍生特征
cluster_features['Value_Density'] = cluster_features['Total_Spending'] / (cluster_features['Purchase_Frequency'] + 1)
cluster_features['Activity_Score'] = 30 - cluster_features['Last_Login_Days_Ago']
cluster_features['Browse_Efficiency'] = cluster_features['Pages_Viewed'] / (cluster_features['Time_Spent_on_Site_Minutes'] + 1)

print(" 聚类特征集构建完成")
print(f"特征数量: {cluster_features.shape[1]}")
print(f"样本数量: {cluster_features.shape[0]}")

 聚类特征集构建完成
特征数量: 11
样本数量: 1000

# 使用RobustScaler标准化（对异常值更稳健）
from sklearn.preprocessing import RobustScaler

scaler = RobustScaler()
features_scaled = scaler.fit_transform(cluster_features)

print(" 数据标准化完成")

# 评估不同K值（2-10）的聚类效果
k_range = range(2, 11)
metrics_results = {
    'K': [],
    'Inertia': [],
    'Silhouette': [],
    'Davies_Bouldin': [],
    'Calinski_Harabasz': []
}

print("="*70)
print(" 正在评估不同K值的聚类效果...")
print("="*70)

for k in k_range:
    # 训练K-Means模型
    kmeans = KMeans(n_clusters=k, random_state=42, n_init=10, max_iter=300)
    labels = kmeans.fit_predict(scaled_features)
    
    # 计算各项评估指标
    inertia = kmeans.inertia_
    silhouette = silhouette_score(scaled_features, labels)
    db_score = davies_bouldin_score(scaled_features, labels)
    ch_score = calinski_harabasz_score(scaled_features, labels)
    
    # 存储结果
    metrics_results['K'].append(k)
    metrics_results['Inertia'].append(inertia)
    metrics_results['Silhouette'].append(silhouette)
    metrics_results['Davies_Bouldin'].append(db_score)
    metrics_results['Calinski_Harabasz'].append(ch_score)
    
    print(f"K={k}: Inertia={inertia:.2f}, Silhouette={silhouette:.4f}, DB={db_score:.4f}, CH={ch_score:.2f}")


# 创建指标DataFrame
metrics_df = pd.DataFrame(metrics_results)
display(metrics_df)