Chapter 4 第四章 探索性数据分析

学习目标

• 📌 了解探索性数据分析的概念

• 📌 掌握数据转化的常见方法

• 📌 了解数据可视化类型与特点

4.1 数据转换

4.1.1 Z-score标准化

4.1.2 归一化

4.1.3 对数化

4.1.4 词频标准化

4.1.5 描述性统计

4.1.5.1 中心趋势度

4.1.5.2 数据分布

4.2 数据可视化

4.2.1 数据可视化工具

4.2.2 R语言绘图程序包

4.2.3 ggplot2图形语法

4.2.4 数据可视化基本图类

4.2.4.1 柱状图

4.2.4.2 箱线图和小提琴图

4.2.4.3 直方图

4.2.4.4 密度图

4.2.4.5 瓦片图

4.3 课堂任务

# 数据处理函数大合集
library(tidyverse)
########################################
# ✔ dplyr     1.1.3     ✔ readr     2.1.4
# ✔ forcats   1.0.0     ✔ stringr   1.5.1
# ✔ ggplot2   3.4.4     ✔ tibble    3.2.1
# ✔ lubridate 1.9.3     ✔ tidyr     1.3.0
# ✔ purrr     1.0.2  
#########################################
#用于获取简奥斯丁小说数据
library(janeaustenr)
#用于文本分词
library(tidytext)

任务4.1 数据标准化与归一化

  计算每个词汇的情感得分与抽象度的乘积作为新特征Emotion_Abstractness_Product。计算每个词汇的词频与意象度的差作为新特征Frequency_Imagery_Difference。对词频特征进行Z-score标准化，并创建新特征Frequency_Zscore。对词汇抽象度进行Min-Max归一化，并创建新特征Abstractness_Normalized。对词频特征进行对数化，并创建新特征Frequency_Log。

# 创建含有真实汉语词汇的数据集
set.seed(123)  # 设置随机种子以确保可重复性
df_vocabulary <- data.frame(Word = c("苹果", "梨子", "香蕉", "草莓", 
                                     "桃子", "西瓜", "橙子", "柚子", "葡萄", "芒果",
                                 "书", "笔", "课本", "教室", "黑板", 
                                 "学生", "老师", "考试", "作业", "校园",
                                 "美食", "旅游", "音乐", "电影", 
                                 "运动", "阅读", "绘画", "写作", "摄影", "游戏",
                                 "猫", "狗", "兔子", "鱼", "鸟",
                                 "大象", "狮子", "熊猫", "猴子", "蛇",
                                 "早晨", "中午", "下午", "晚上", 
                                 "春天", "夏天", "秋天", "冬天", "天气", "季节",
                                 "篮球", "足球", "乒乓球", "网球",
                                 "羽毛球", "游泳", "滑雪", "跑步", "健身", "瑜伽"),
                        Frequency = sample(1:1000, 60, replace = TRUE),
                        Abstractness = sample(1:5, 60, replace = TRUE),
                        Imagery = sample(1:5, 60, replace = TRUE),
                        Age_of_Acquisition = sample(3:12, 60, replace = TRUE),
                        Difficulty = sample(1:5, 60, replace = TRUE),
                        Emotion_Score = sample(1:5, 60, replace = TRUE))


# 使用mutate函数创建新特征
df_vocabulary %>% 
  mutate(Emotion_Abstractness_Product = Emotion_Score * Abstractness,
         Frequency_Imagery_Difference = Frequency - Imagery,
         Frequency_Zscore = scale(Frequency),
         Abstractness_Normalized = (Abstractness - min(Abstractness)) / (max(Abstractness) - min(Abstractness)),
         Frequency_Log = log(Frequency))

任务4.2 描述性统计

计算男女生各门成绩的均分和标准差

df.score = data.frame(Student_ID = 1:100,
                    Listening_Score = sample(-5:100, 100, replace = TRUE),
                    Speaking_Score = sample(0:999, 100, replace = TRUE),
                    Reading_Score = sample(0:105, 100, replace = TRUE),
                    Writing_Score = sample(0:100, 100, replace = TRUE),
                    Gender = sample(c("Male", "Female"), 100, replace = TRUE))

# 计算每种母语的学习者的平均学习时长
df.mean.sd <- df.score %>%
  filter(Listening_Score > 0 | Reading_Score < 100)%>%
  group_by(Gender) %>%
  summarise(Listening.mean = mean(Listening_Score),
            Listening.sd = sd(Listening_Score))

任务4.3

austen.emma.word <- austen_books() %>%
  group_by(book) %>%
  mutate(linenumber = row_number(),
         chapter = cumsum(str_detect(text, 
                                     regex("^chapter [\\divxlc]",
                                           ignore_case = TRUE)))) %>%
  unnest_tokens(word, text)%>%
  count(word, sort = TRUE) %>% 
  #筛选Emma这个小说
  filter(book == "Emma")%>%
  mutate(total = sum(n),
         relativefrq = (n/total)*100)%>%
  #显示前5的高频词
  head(5)


austen.word = austen_books() %>%
  group_by(book) %>%
  unnest_tokens(word, text)%>%
  count(word, sort = TRUE) %>% 
  group_by(book)%>%
  mutate(total = sum(n),
         relativefrq = (n/total)*100)%>%
  filter(word %in% austen.emma.word$word)

austen.word %>%
  mutate(word = reorder(word, relativefrq)) %>%
  ggplot(aes(relativefrq, word, fill = book)) +  # fill按book变量着色
  geom_col(position = position_dodge(width = 0.7),  # 关键：平行排列，width控制间距
           width = 0.6) +  # 控制条形的宽度
  scale_fill_brewer(palette = "Set1") +  # 使用区分度好的配色
  labs(y = NULL, x = "Frequency", fill = "Book") +
  theme_minimal() +
  theme(legend.position = "top")  # 图例放在顶部

任务4.4 双变量可视化

naming.df = read.csv('/Users/chenjuqiang/Nutstore Files/310_Tutorial/LanguageDS-MS/data/ch4/pictureNaming.new.csv')%>%
  mutate(LEN = as.factor(LEN),
         Source = as.factor(Source))

table(naming.df$LEN,naming.df$Source)

naming.df%>% 
  count(Source, LEN)%>%
  ggplot(aes(Source, LEN))+
  geom_tile(aes(fill=n))


ggplot(lexdec)+
  geom_count(aes(Class, Correct))

# 条形图
naming.df%>% 
  group_by(LEN)%>%
  summarise(mean = mean(RT_harm))%>%
  ggplot(aes(LEN, mean))+
  geom_bar(stat="identity")

naming.df%>% 
  group_by(LEN)%>%
  summarise(mean = mean(RT_harm),
            sd = sd(RT_harm))%>%
  ggplot(aes(LEN, mean, fill = LEN))+
  geom_bar(stat="identity")+
  geom_errorbar(aes(ymin = mean - sd,
                    ymax = mean + sd),
                width = .2, size = 0.7, position = position_dodge(.9))


# 箱型图
naming.df%>% 
 ggplot(., aes(x=LEN, y=RT_harm))+
  geom_boxplot()

# 小提琴图
naming.df%>% 
 ggplot(., aes(x=LEN, y=RT_harm))+
    geom_violin()

# 密度图
ggplot(naming.df, aes(x = RT_harm, y =  ..density..))+
  geom_freqpoly(aes(linetype = LEN), binwidth = 100)


# 双连续变量

ggplot(naming.df)+
  geom_point(aes(RT_harm, FAM))