image.png

一.数据集简介

tmdb_5000_movies和tmdb_5000_credits这两个数据集所包含的特征

tmdb_5000_movies：

homepage，id，original_title，overview，popularity，production_companies，production_countries，release_date，spoken_languages，status，tagline，vote_average

tmdb_5000_credits：

movie_id，title，cast，crew

二.对原始数据集进行预处理

import json

import pandas as pd

#load_tmdb_movies用于载入movie数据集

def load_tmdb_movies(path):

df = pd.read_csv(path)

df['release_date'] = pd.to_datetime(df['release_date']).apply(lambda x: x.date())

json_columns = ['genres', 'keywords', 'production_countries',

'production_companies', 'spoken_languages']

for column in json_columns:

df[column] = df[column].apply(json.loads)

return df

#load_tmdb_credits用于载入credits数据集

def load_tmdb_credits(path):

df = pd.read_csv(path)

json_columns = ['cast', 'crew']

for column in json_columns:

df[column] = df[column].apply(json.loads)

return df

#用于更改原始数据中的列名

TMDB_TO_IMDB_SIMPLE_EQUIVALENCIES = {

'budget': 'budget',

'genres': 'genres',

'revenue': 'gross',

'title': 'movie_title',

'runtime': 'duration',

'original_language': 'language',

'keywords': 'plot_keywords',

'vote_count': 'num_voted_users'}

#检索函数

def safe_access(container, index_values):

result = container

try:

for idx in index_values:

result = result[idx]

return result

except IndexError or KeyError:

return pd.np.nan

#关键字处理函数，可以将关键字用"|"隔开

def pipe_flatten_names(keywords):

return '|'.join([x['name'] for x in keywords])

#对原始数据

def convert_to_original_format(movies, credits):

tmdb_movies = movies.copy()

tmdb_movies.rename(columns=TMDB_TO_IMDB_SIMPLE_EQUIVALENCIES, inplace=True)

tmdb_movies['title_year'] = pd.to_datetime(tmdb_movies['release_date']).apply(lambda x: x.year)

tmdb_movies['country'] = tmdb_movies['production_countries'].apply(lambda x: safe_access(x, [0, 'name']))

tmdb_movies['language'] = tmdb_movies['spoken_languages'].apply(lambda x: safe_access(x, [0, 'name']))

tmdb_movies['director_name'] = credits['crew'].apply(get_director)

tmdb_movies['actor_1_name'] = credits['cast'].apply(lambda x: safe_access(x, [1, 'name']))

tmdb_movies['actor_2_name'] = credits['cast'].apply(lambda x: safe_access(x, [2, 'name']))

tmdb_movies['actor_3_name'] = credits['cast'].apply(lambda x: safe_access(x, [3, 'name']))

tmdb_movies['genres'] = tmdb_movies['genres'].apply(pipe_flatten_names)

tmdb_movies['plot_keywords'] = tmdb_movies['plot_keywords'].apply(pipe_flatten_names)

return tmdb_movies

#导入必要的包

import numpy as np

import matplotlib as mpl

import matplotlib.pyplot as plt

import seaborn as sns

import math, nltk, warnings

from nltk.corpus import wordnet

from sklearn import linear_model

from sklearn.neighbors import NearestNeighbors

from fuzzywuzzy import fuzz

from wordcloud import WordCloud, STOPWORDS

plt.rcParams["patch.force_edgecolor"] = True

plt.style.use('fivethirtyeight')

mpl.rc('patch', edgecolor = 'dimgray', linewidth=1)

from IPython.core.interactiveshell import InteractiveShell

InteractiveShell.ast_node_interactivity = "last_expr"

pd.options.display.max_columns = 50

%matplotlib inline

warnings.filterwarnings('ignore')

PS = nltk.stem.PorterStemmer()

#载入原始数据并进行预处理

credits = load_tmdb_credits("tmdb_5000_credits.csv")

movies = load_tmdb_movies("tmdb_5000_movies.csv")

df_initial = convert_to_original_format(movies, credits)

print('Shape:',df_initial.shape)

tab_info = pd.DataFrame(df_initial.dtypes).T.rename(index={0: 'column type'})

tab_info = tab_info.append(pd.DataFrame(df_initial.isnull().sum()).T.rename(index={0:'null values'}))

tab_info=tab_info.append(pd.DataFrame(df_initial.isnull().sum()/df_initial.shape[0]*100).T. rename(index={0:'null values (%)'}))

从输出结果中可以看出df_initial的shape为(4803, 26)。另外，tab_info表示为(因为是截图所以还有好几列被省略了)：

image.png

下一篇将会进入正式的推荐系统设计环节~