types_train = {
  'id': np.dtype(int), 
  'click': np.dtype(int),    #是否點(diǎn)擊,1表示被點(diǎn)擊,0表示沒(méi)被點(diǎn)擊
  'hour': np.dtype(int),    #廣告被展現(xiàn)的日期+時(shí)間
  'C1': np.dtype(int),     #匿名分類變量
  'banner_pos': np.dtype(int), #廣告位置
  'site_id': np.dtype(str),   #站點(diǎn)Id
  'site_domain': np.dtype(str), #站點(diǎn)域名
  'site_category': np.dtype(str), #站點(diǎn)分類
  'app_id': np.dtype(str),    # appId 
  'app_domain': np.dtype(str),  # app域名
  'app_category': np.dtype(str), # app分類
  'device_id': np.dtype(str),   #設(shè)備Id
  'device_ip': np.dtype(str),   #設(shè)備Ip
  'device_model': np.dtype(str), #設(shè)備型號(hào)
  'device_type': np.dtype(int),  #設(shè)備型號(hào)
  'device_conn_type': np.dtype(int),
  'C14': np.dtype(int),  #匿名分類變量
  'C15': np.dtype(int),  #匿名分類變量
  'C16': np.dtype(int),  #匿名分類變量
  'C17': np.dtype(int),  #匿名分類變量
  'C18': np.dtype(int),  #匿名分類變量
  'C19': np.dtype(int),  #匿名分類變量
  'C20': np.dtype(int),  #匿名分類變量
  'C21':np.dtype(int)  #匿名分類變量
}
 
n = 40428967 #數(shù)據(jù)集中的記錄總數(shù)
sample_size = 1000000
skip_values = sorted(random.sample(range(1,n), n-sample_size)) 
parse_date = lambda val : pd.datetime.strptime(val, '%y%m%d%H')
 
with gzip.open('./data/ctr/train.gz') as f:
  train = pd.read_csv(f, parse_dates = ['hour'], date_parser = parse_date, dtype=types_train, skiprows = skip_values)
print(len(train))
train.head()

print(train['click'].value_counts())
print()
print(train['click'].value_counts()/len(train))
 
sns.countplot(x='click',data=train, palette='hls')
plt.show()

print(train.hour.describe())
 
train.groupby('hour').agg({'click':'sum'}).plot(figsize=(12,6))
plt.ylabel('點(diǎn)擊量')
plt.title('時(shí)間和點(diǎn)擊量')

train['time'] = train.hour.apply(lambda x: x.hour)
train.sample(5)

train.groupby('time').agg({'click':'sum'}).plot(figsize=(12,6),grid=True)
plt.ylabel('點(diǎn)擊次數(shù)')
plt.title('時(shí)間和點(diǎn)擊量')

train.groupby(['time', 'click']).size().unstack().plot(kind='bar', figsize=(12,6))
plt.ylabel('數(shù)量')
plt.title('展現(xiàn)量與點(diǎn)擊量');

df_click = train[train['click'] == 1]
df_hour = train[['time','click']].groupby(['time']).count().reset_index()
df_hour = df_hour.rename(columns={'click': 'impressions'})
df_hour['clicks'] = df_click[['time','click']].groupby(['time']).count().reset_index()['click']
df_hour['CTR'] = df_hour['clicks']/df_hour['impressions']*100
df_hour.head()

plt.figure(figsize=(12,6))
sns.barplot(y='CTR', x='time', data=df_hour)
plt.title('點(diǎn)擊率的時(shí)間分布')

train['day_of_week'] = train['hour'].apply(lambda val: val.weekday_name)
cats = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
train.groupby('day_of_week').agg({'click':'sum'}).reindex(cats).plot(figsize=(12,6))
ticks = list(range(0, 7, 1)) 
labels = "周一 周二 周三 周四 周五 周六 周日".split()
plt.xticks(ticks, labels)
plt.title('星期的點(diǎn)擊量')

train.groupby(['day_of_week','click']).size().unstack().reindex(cats).plot(kind='bar', title="Day of the Week", figsize=(12,6))
ticks = list(range(0, 7, 1)) 
labels = "周一 周二 周三 周四 周五 周六 周日".split()
plt.xticks(ticks, labels)
plt.title('星期的展現(xiàn)量和點(diǎn)擊量分布')

df_click = train[train['click'] == 1]
df_dayofweek = train[['day_of_week','click']].groupby(['day_of_week']).count().reset_index()
df_dayofweek = df_dayofweek.rename(columns={'click': 'impressions'})
df_dayofweek['clicks'] = df_click[['day_of_week','click']].groupby(['day_of_week']).count().reset_index()['click']
df_dayofweek['CTR'] = df_dayofweek['clicks']/df_dayofweek['impressions']*100
 
plt.figure(figsize=(12,6))
sns.barplot(y='CTR', x='day_of_week', data=df_dayofweek, order=['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'])
plt.title('星期的點(diǎn)擊率')

print(train.C1.value_counts()/len(train))

C1_values = train.C1.unique()
C1_values.sort()
ctr_avg_list=[]
for i in C1_values:
  ctr_avg=train.loc[np.where((train.C1 == i))].click.mean()
  ctr_avg_list.append(ctr_avg)
  print(" C1 value: {}, 點(diǎn)擊率: {}".format(i,ctr_avg))

train.groupby(['C1', 'click']).size().unstack().plot(kind='bar', figsize=(12,6), title='C1 展現(xiàn)量和點(diǎn)擊量分布');

df_c1 = train[['C1','click']].groupby(['C1']).count().reset_index()
df_c1 = df_c1.rename(columns={'click': 'impressions'})
df_c1['clicks'] = df_click[['C1','click']].groupby(['C1']).count().reset_index()['click']
df_c1['CTR'] = df_c1['clicks']/df_c1['impressions']*100
 
plt.figure(figsize=(12,6))
sns.barplot(y='CTR', x='C1', data=df_c1)
plt.title('C1的點(diǎn)擊率分布')

banner_pos = train.banner_pos.unique()
banner_pos.sort()
ctr_avg_list=[]
for i in banner_pos:
  ctr_avg=train.loc[np.where((train.banner_pos == i))].click.mean()
  ctr_avg_list.append(ctr_avg)
  print(" banner 位置: {}, 點(diǎn)擊率: {}".format(i,ctr_avg))

train.groupby(['banner_pos', 'click']).size().unstack().plot(kind='bar', figsize=(12,6), title='banner 位置的廣告展現(xiàn)量和點(diǎn)擊量的分布')

df_banner = train[['banner_pos','click']].groupby(['banner_pos']).count().reset_index()
df_banner = df_banner.rename(columns={'click': 'impressions'})
df_banner['clicks'] = df_click[['banner_pos','click']].groupby(['banner_pos']).count().reset_index()['click']
df_banner['CTR'] = df_banner['clicks']/df_banner['impressions']*100
sort_banners = df_banner.sort_values(by='CTR',ascending=False)['banner_pos'].tolist()
plt.figure(figsize=(12,6))
sns.barplot(y='CTR', x='banner_pos', data=df_banner, order=sort_banners)
plt.title('banner 位置的點(diǎn)擊率的分布')

train[['device_type','click']].groupby(['device_type','click']).size().unstack().plot(kind='bar', title='設(shè)備類型')

df_click[df_click['device_type']==1].groupby(['time', 'click'])\
                   .size().unstack()\
                   .plot(kind='bar', title="設(shè)備1的點(diǎn)擊量分布", figsize=(12,6))

device_type_click = df_click.groupby('device_type').agg({'click':'sum'}).reset_index()
device_type_impression = train.groupby('device_type').agg({'click':'count'}).reset_index().rename(columns={'click': 'impressions'})
merged_device_type = pd.merge(left = device_type_click , right = device_type_impression, how = 'inner', on = 'device_type')
 
merged_device_type['CTR'] = merged_device_type['click'] / merged_device_type['impressions']*100
 
merged_device_type

def convert_obj_to_int(self):
  
  object_list_columns = self.columns
  object_list_dtypes = self.dtypes
  new_col_suffix = '_int'
  for index in range(0,len(object_list_columns)):
    if object_list_dtypes[index] == object :
      self[object_list_columns[index]+new_col_suffix] = self[object_list_columns[index]].map( lambda x: hash(x))
      self.drop([object_list_columns[index]],inplace=True,axis=1)
  return self
train = convert_obj_to_int(train)
train.head()

X_train = train.loc[:, train.columns != 'click']
y_target = train.click.values
 
msk = np.random.rand(len(X_train)) < 0.8
lgb_train = lgb.Dataset(X_train[msk], y_target[msk])
lgb_eval = lgb.Dataset(X_train[~msk], y_target[~msk], reference=lgb_train)
 
# 配置模型參數(shù)
params = {
  'task': 'train',
  'boosting_type': 'gbdt',
  'objective': 'binary',
  'metric': { 'binary_logloss'},
  'num_leaves': 31, # 每棵樹(shù)的默認(rèn)葉子數(shù)
  'learning_rate': 0.08,
  'feature_fraction': 0.7, # 將在訓(xùn)練每棵樹(shù)之前選擇70％的特征
  'bagging_fraction': 0.3, #隨機(jī)選擇30%的特征。
  'bagging_freq': 5, # 每5次迭代執(zhí)行bagging
  'verbose': 0
}
 
print('開(kāi)始訓(xùn)練...')
 
gbm = lgb.train(params,
        lgb_train,
        num_boost_round=4000,
        valid_sets=lgb_eval,
        early_stopping_rounds=500)

def run_default_test(train, test, features, target, random_state=0):
  eta = 0.1
  max_depth = 5
  subsample = 0.8
  colsample_bytree = 0.8
  params = {
    "objective": "binary:logistic",
    "booster" : "gbtree",
    "eval_metric": "logloss",
    "eta": eta,
    "max_depth": max_depth,
    "subsample": subsample,
    "colsample_bytree": colsample_bytree,
    "silent": 1,
    "seed": random_state
  }
  num_boost_round = 260
  early_stopping_rounds = 20
  test_size = 0.2
 
  X_train, X_valid = train_test_split(train, test_size=test_size, random_state=random_state)
  y_train = X_train[target]
  y_valid = X_valid[target]
  dtrain = xgb.DMatrix(X_train[features], y_train)
  dvalid = xgb.DMatrix(X_valid[features], y_valid)
  watchlist = [(dtrain, 'train'), (dvalid, 'eval')]
  gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=True)
 
 
features = ['C1', 'banner_pos', 'device_type', 'device_conn_type', 'C14',
    'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21', 'time',
    'site_id_int', 'site_domain_int', 'site_category_int', 'app_id_int',
    'app_domain_int', 'app_category_int', 'device_id_int', 'device_ip_int',
    'device_model_int', 'day_of_week_int']
run_default_test(train, y_target, features, 'click')