02_Packages.R

# ============================================================
# Packages for Data Transformation
# Nevada Bioinformatics Center
# ============================================================


# ============================================================
# INTRODUCTION
# ============================================================

# In this lesson, we'll learn about R packages that make data manipulation
# easier and more intuitive. While base R is powerful, packages like dplyr
# and tidyr provide functions specifically designed for common data analysis tasks.
#
# Packages are like toolboxes -- base R gives you basic tools, but specialized
# packages give you power tools that make specific jobs much easier and faster.


# ============================================================
# R PACKAGES
# ============================================================

# R packages are collections of functions, data, and documentation that
# extend R's capabilities. The tidyverse is a collection of packages designed
# to work together seamlessly for data science tasks.

# --- Installing and Loading Packages ---
# Before we can use a package, we need to:
#   1. Install it (only once)
#   2. Load it (every session)

# Install the tidyverse (only need to do this ONCE)
install.packages("tidyverse")


# ============================================================
# WHAT IS THE TIDYVERSE?
# ============================================================

# The tidyverse is a collection of R packages designed to make data science
# tasks easier, more consistent, and more readable. These packages are built
# to work together and share a common philosophy and grammar.
#
# The most important tidyverse packages:
#   dplyr   -- data manipulation (filtering, selecting, summarizing, etc.)
#   tidyr   -- reshaping and tidying data
#   ggplot2 -- data visualization
#   readr   -- reading data files (like CSVs)
#   tibble  -- modern, user-friendly data frames
#   stringr -- working with text (strings)
#   forcats -- working with categorical data (factors)
#
# All tidyverse packages:
#   - Work with tibbles (a modern version of data frames)
#   - Use verbs like select(), filter(), mutate(), summarize()
#   - Use the pipe operator (%>%) to chain steps in a readable way
#
# Why learn the tidyverse?
#   Once you learn the core tidyverse verbs and the pipe operator, you can
#   use them across almost all tidyverse packages -- making it much easier to
#   learn new packages and work with data consistently.
#
# Tip: The tidyverse is not just for advanced users -- it's designed to make
# R easier and more intuitive for everyone!


# Load tidyverse for this session
library(tidyverse)

# Load our practice data
data(mtcars)
data(iris)


# ============================================================
# LEARNING dplyr
# ============================================================

# dplyr provides a grammar of data manipulation.
# It gives us a set of functions for the most common data manipulation challenges.
#
# The main dplyr functions are:
#   select()    -- pick columns
#   filter()    -- pick rows
#   mutate()    -- create new columns
#   arrange()   -- reorder rows
#   group_by()  -- group data for analysis
#   summarize() -- calculate summary statistics


# --- select() -- Choosing Columns ---
#
# select() helps you pick out just the columns you want to keep.
#
# Why use select()?
#   - To simplify your data and focus only on the variables you care about
#   - To make your code and results easier to read
#   - To avoid mistakes by removing unnecessary columns

# Look at the original mtcars data
head(mtcars)

# Select just a few columns
mtcars_simple <- select(mtcars, mpg, hp, wt)
head(mtcars_simple)

# Select columns by position
select(mtcars, 1:3)

# Select all columns except certain ones
select(mtcars, -cyl, -disp)

# Select columns that start with a letter
select(mtcars, starts_with("c"))   # cyl and carb


# --- filter() -- Choosing Rows ---
#
# filter() keeps only rows that meet certain conditions.
# This is like subsetting but with cleaner syntax.

# Filter cars with high fuel efficiency
high_mpg <- filter(mtcars, mpg > 25)
high_mpg

# Filter with multiple conditions (AND)
efficient_powerful <- filter(mtcars, mpg > 20, hp > 100)
head(efficient_powerful)

# Filter with OR conditions
filter(mtcars, mpg > 30 | hp > 250)

# Filter based on categorical data
filter(iris, Species == "setosa")


# --- mutate() -- Creating New Columns ---
#
# mutate() adds new columns to your data frame, often based on existing columns.
# This is perfect for calculating new variables.

# Create a new column for horsepower per weight
mtcars_new <- mutate(mtcars, hp_per_weight = hp / wt)
head(select(mtcars_new, hp, wt, hp_per_weight))

# Create multiple new columns at once
mtcars_enhanced <- mutate(mtcars,
                          hp_per_weight = hp / wt,
                          is_efficient  = mpg > 20,
                          weight_kg     = wt * 453.592)   # convert to kg

head(select(mtcars_enhanced, mpg, hp, wt, hp_per_weight, is_efficient, weight_kg))

# mutate() can use columns you just created in the same call
mtcars_mutated <- mutate(mtcars,
                         weight_tons = wt / 2,
                         hp_per_ton  = hp / (wt / 2))
head(select(mtcars_mutated, hp, wt, weight_tons, hp_per_ton))


# --- arrange() -- Sorting Rows ---
#
# arrange() sorts your data based on one or more columns.
# By default it sorts in ascending order.

# Sort by miles per gallon (lowest to highest)
head(arrange(mtcars, mpg))

# Sort by miles per gallon (highest to lowest)
head(arrange(mtcars, desc(mpg)))

# Sort by multiple columns
head(arrange(mtcars, cyl, desc(mpg)))


# --- count() -- Counting Occurrences ---
#
# count() is a quick way to count how many times each value appears in a column.

# Count cars by number of cylinders
count(mtcars, cyl)

# Count by multiple variables
count(mtcars, cyl, gear)

# Count and sort by most common
count(mtcars, cyl, sort = TRUE)

# With iris data
count(iris, Species)


# ============================================================
# PIPES (%>%) -- CHAINING OPERATIONS
# ============================================================

# The pipe operator %>% lets you chain multiple operations together.
# Read it as "then do".
#
# What is a pipe?
#   The pipe takes the result from the left side and passes it as the first
#   argument to the function on the right side. This allows you to write a
#   sequence of steps in the order you would describe them.
#
# This makes your code:
#   - Easier to read and write
#   - Avoids deeply nested parentheses
#   - Lets you build up your analysis step by step
#
# How to read a pipe chain:
#   mtcars %>% select(mpg, hp) %>% filter(mpg > 20)
#   means: "Take mtcars, THEN select mpg and hp, THEN keep rows where mpg > 20."
#
# Tip: You can break a long pipe chain into multiple lines for clarity.
#      RStudio will automatically indent the next line after a pipe.
#
# Common mistake: Don't forget to load tidyverse or dplyr, or %>% won't work!


# Without pipes (hard to read -- working from inside out)
result <- arrange(
  filter(
    select(mtcars, mpg, hp, wt),
    mpg > 20
  ),
  desc(hp)
)
head(result)

# With pipes (much clearer -- working from left to right)
mtcars %>%
  select(mpg, hp, wt) %>%
  filter(mpg > 20) %>%
  arrange(desc(hp)) %>%
  head()


# --- group_by() and summarize() -- Grouped Analysis ---
#
# These functions work together to perform calculations on groups within data.
#
#   group_by()  -- divides your data into groups
#   summarize() -- calculates statistics for each group

# Calculate average mpg by number of cylinders
mtcars %>%
  group_by(cyl) %>%
  summarize(avg_mpg = mean(mpg))

# Multiple summary statistics at once
mtcars %>%
  group_by(cyl) %>%
  summarize(
    count      = n(),           # count of cars in each group
    avg_mpg    = mean(mpg),     # average mpg
    max_hp     = max(hp),       # maximum horsepower
    avg_weight = mean(wt)       # average weight
  )

# Group by multiple variables
mtcars %>%
  group_by(cyl, gear) %>%
  summarize(
    count   = n(),
    avg_mpg = mean(mpg)
  )


# --- Putting It All Together ---
# A more complex example combining multiple dplyr functions.

# Analyze efficient cars (>20 mpg) by cylinder count
efficient_car_analysis <- mtcars %>%
  filter(mpg > 20) %>%                          # only efficient cars
  mutate(hp_per_weight = hp / wt) %>%           # create efficiency ratio
  group_by(cyl) %>%                             # group by cylinders
  summarize(
    car_count      = n(),
    avg_mpg        = round(mean(mpg), 1),
    avg_hp         = round(mean(hp), 1),
    avg_efficiency = round(mean(hp_per_weight), 2)
  ) %>%
  arrange(desc(avg_mpg))                        # sort by mpg descending

efficient_car_analysis


# ============================================================
# DATA WRANGLING WITH tidyr
# ============================================================

# tidyr helps you reshape your data. Most analysis and visualization functions
# expect data in a specific format, so knowing how to reshape data is crucial.


# --- Long vs Wide Data ---
#
# Data can be organized in two main ways:
#
# WIDE data -- each variable has its own column:
#   Country   2018   2019   2020
#   USA        100    110    105
#   Canada      90     95     92
#
# LONG data -- each observation is a separate row:
#   Country   Year   Value
#   USA       2018   100
#   USA       2019   110
#   USA       2020   105
#   Canada    2018    90
#   Canada    2019    95
#   Canada    2020    92
#
# When to use each:
#   Wide format -- easier for humans to read; some statistical tests
#   Long format -- better for ggplot2 plots and grouped analysis


# --- Pivot Functions ---
#
# tidyr provides two main functions for reshaping data:
#   pivot_longer() -- converts wide data to long data
#   pivot_wider()  -- converts long data to wide data
#
# Note: Many tidyverse functions return a TIBBLE instead of a regular data frame.
#
# What is a tibble?
#   A tibble is a modern version of a data frame. It looks and behaves almost
#   the same, but:
#     - Prints more nicely (shows only first 10 rows, fits your screen)
#     - Never automatically changes data types
#     - More consistent and user-friendly
#
# You can check with class(). To convert back to a regular data frame:
# grades_long_df <- as.data.frame(grades_long)


# Create sample wide data (grades for different subjects)
grades_wide <- data.frame(
  student = c("Alice", "Bob", "Charlie", "Diana"),
  math    = c(85, 78, 92, 88),
  science = c(90, 82, 89, 94),
  english = c(88, 85, 78, 91)
)
grades_wide


# --- pivot_longer() -- Wide to Long ---
#
# pivot_longer() takes multiple columns and combines them into two columns:
# one for the variable names and one for the values.

# Convert wide to long
grades_long <- grades_wide %>%
  pivot_longer(
    cols      = c(math, science, english),   # columns to pivot
    names_to  = "subject",                   # name for the new "names" column
    values_to = "grade"                      # name for the new "values" column
  )
grades_long

# Alternative syntax -- select columns by excluding others
grades_wide %>%
  pivot_longer(
    cols      = -student,    # all columns EXCEPT student
    names_to  = "subject",
    values_to = "grade"
  )

# Or use column positions
grades_wide %>%
  pivot_longer(
    cols      = 2:4,         # columns 2 through 4
    names_to  = "subject",
    values_to = "grade"
  )

# Now we can do statistics on the long data
grades_long %>%
  group_by(subject) %>%
  summarize(avg_grade = mean(grade))

# Find the best student in each subject
grades_long %>%
  group_by(subject) %>%
  slice_max(grade, n = 1)   # get the top grade in each subject


# --- pivot_wider() -- Long to Wide ---
#
# pivot_wider() does the opposite -- takes two columns and spreads them
# into multiple columns.

# Convert back to wide format
grades_back_to_wide <- grades_long %>%
  pivot_wider(
    names_from  = subject,   # column containing new column names
    values_from = grade      # column containing values
  )
grades_back_to_wide

# A more complex example: student performance over time
performance <- data.frame(
  student = rep(c("Alice", "Bob"), each = 3),
  quarter = rep(c("Q1", "Q2", "Q3"), times = 2),
  grade   = c(85, 87, 90, 78, 80, 83)
)
performance

# Pivot wider to see each quarter as a column
performance_wide <- performance %>%
  pivot_wider(
    names_from  = quarter,
    values_from = grade
  )
performance_wide


# --- When to Use Each Format ---
#
# Use WIDE format when you want to:
#   - Compare values across categories side by side
#   - Calculate correlations between variables
#   - Create tables for reports
#   - Use functions that expect each variable in its own column
#
# Use LONG format when you want to:
#   - Create plots with ggplot2 (especially for color/grouping by category)
#   - Calculate statistics by group
#   - Perform operations on multiple columns at once

# Wide format is good for correlation analysis
cor(grades_wide[, -1])   # correlation between subjects (excluding student names)

# Long format is good for grouped analysis
grades_long %>%
  group_by(subject) %>%
  summarize(
    avg_grade   = mean(grade),
    min_grade   = min(grade),
    max_grade   = max(grade),
    grade_range = max_grade - min_grade
  )


# ============================================================
# SAVING YOUR RESULTS AND OUTPUTS
# ============================================================

# Now that you've learned to manipulate and reshape data, it's important to
# know how to SAVE your results for future use or sharing.
# This is why we set up folders like output/ and figures/ at the start.


# --- Saving and Loading a Dataset ---
#
# write.csv() saves a data frame to a CSV file on your computer.
# read.csv() loads a CSV file back into R as a data frame.
#
#   file = "output/my_data.csv"   -- path and file name
#   row.names = FALSE             -- avoids saving row numbers as an extra column

# Save a data frame to a CSV file
write.csv(my_data, file = "output/my_data.csv", row.names = FALSE)

# Later, or in a new R session, load it back:
my_loaded_data <- read.csv("output/my_data.csv")


# --- Saving Plots as Image Files ---
#
# You can save plots as PNG or JPEG files for reports or presentations.
# This is a three-step process:
#
#   Step 1: Open a graphics device (png() or jpeg()) and give it a file name.
#           This tells R: "Get ready to save a plot as an image file."
#
#   Step 2: Create your plot as usual with plot(), barplot(), etc.
#           The plot won't appear in the RStudio Plots pane -- it's being
#           "drawn" into the image file.
#
#   Step 3: Close the device with dev.off() to finish saving the file.
#           IMPORTANT: Don't forget this step or your file won't be saved properly!
#
# Think of it like: "start recording" -> do your work -> "stop recording and save"

# Save a plot as a PNG image
png("figures/my_plot.png", width = 600, height = 400)
plot(mtcars$hp, mtcars$mpg)
dev.off()


# --- Example: Save a Filtered Dataset ---
# Create a new dataset: cars with mpg > 25
efficient_cars <- filter(mtcars, mpg > 25)

# Save to CSV in the output folder
write.csv(efficient_cars, file = "output/efficient_cars.csv", row.names = TRUE)

# Now you'll find efficient_cars.csv in your output/ folder!


# --- Example: Save a Basic Scatter Plot ---

png("figures/mpg_vs_hp.png", width = 600, height = 400)
plot(mtcars$hp, mtcars$mpg,
     main = "Fuel Efficiency vs Horsepower",
     xlab = "Horsepower",
     ylab = "Miles per Gallon",
     col  = "blue", pch = 19)
dev.off()

# This creates mpg_vs_hp.png in your figures/ folder.


# --- Example: Save a Summary Table ---

# Create a summary table: average mpg by number of cylinders
mpg_by_cyl <- mtcars %>%
  group_by(cyl) %>%
  summarize(avg_mpg = mean(mpg))

# Save the summary table as a CSV
write.csv(mpg_by_cyl, file = "output/mpg_by_cyl.csv", row.names = FALSE)

# Tip: Always check your output/ and figures/ folders after running your code
# to make sure your files were saved as expected!


# ============================================================
# EXERCISES
# ============================================================

# How to approach these exercises:
# - Try each one on your own first
# - Don't worry about making mistakes -- that's how you learn!
# - Create a script for your code and save your results


# --- Exercise 1: Basic dplyr Operations ---
# Using the mtcars dataset, create variables to answer the following:

# 1. Select only the columns: mpg, cyl, hp, and wt
# Your code here:

# 2. Filter cars that have more than 6 cylinders AND more than 200 horsepower
# Your code here:

# 3. Create a new column called "power_to_weight" (hp divided by wt)
# Your code here:

# 4. Arrange cars by mpg from highest to lowest
# Your code here:

# 5. Count how many cars have each number of cylinders
# Your code here:

# 6. Save your power_to_weight dataset as a CSV file in your output folder
# Your code here:


# --- Exercise 2: Pipes and Grouped Analysis ---
# Using the mtcars dataset, use pipes to:
#   1. Filter cars with mpg > 15
#   2. Create a new column hp_per_weight = hp / wt
#   3. Group by number of cylinders
#   4. Summarize to get: count of cars, average mpg, average hp_per_weight
#   5. Arrange by average mpg
#   6. Save this summary table as "cylinder_analysis.csv" in your output folder

# Your code here:


# --- Exercise 3: Pivoting Practice ---
# Temperature data for different cities
temp_data <- data.frame(
  city = c("Las Vegas", "Reno", "Carson City"),
  jan  = c(45, 32, 35),
  apr  = c(65, 55, 58),
  jul  = c(95, 85, 88),
  oct  = c(70, 60, 63)
)
temp_data

# 1. Convert this wide data to long format.
#    Call the new columns "month" and "temperature".
# Your code here:

# 2. From your long format data, calculate the average temperature for each month.
# Your code here:

# 3. Save your long format temperature data as "temp_long.csv"
# Your code here:


# --- Solutions ---

# Exercise 1 Solutions

# 1. Select columns
selected_cars <- select(mtcars, mpg, cyl, hp, wt)

# 2. Filter cars
powerful_cars <- filter(mtcars, cyl > 6, hp > 200)

# 3. Create new column
cars_with_ratio <- mutate(mtcars, power_to_weight = hp / wt)

# 4. Arrange by mpg
cars_by_mpg <- arrange(mtcars, desc(mpg))

# 5. Count cylinders
cyl_counts <- count(mtcars, cyl)

# 6. Save the power_to_weight dataset
write.csv(cars_with_ratio, file = "output/cars_with_power_ratio.csv", row.names = TRUE)


# Exercise 2 Solution

cylinder_analysis <- mtcars %>%
  filter(mpg > 15) %>%
  mutate(hp_per_weight = hp / wt) %>%
  group_by(cyl) %>%
  summarize(
    car_count          = n(),
    avg_mpg            = mean(mpg),
    avg_hp_per_weight  = mean(hp_per_weight)
  ) %>%
  arrange(avg_mpg)

write.csv(cylinder_analysis, file = "output/cylinder_analysis.csv", row.names = FALSE)


# Exercise 3 Solutions

# 1. Wide to long
temp_long <- temp_data %>%
  pivot_longer(
    cols      = -city,
    names_to  = "month",
    values_to = "temperature"
  )
temp_long

# 2. Average temperature by month
avg_temp_by_month <- temp_long %>%
  group_by(month) %>%
  summarize(avg_temp = mean(temperature))
avg_temp_by_month

# 3. Save long format data
write.csv(temp_long, file = "output/temp_long.csv", row.names = FALSE)