This R Notebook is the complement to my blog post A Visual Overview of Stack Overflow’s Question Tags.

This notebook is licensed under the MIT License. If you use the code or data visualization designs contained within this notebook, it would be greatly appreciated if proper attribution is given back to this notebook and/or myself. Thanks! :)

1 Setup

library(tidyverse)

[30m── [1mAttaching packages[22m ─────────────────────────────────────────────────────────────────── tidyverse 1.2.1 ──[39m
[30m[32m✔[30m [34mggplot2[30m 2.2.1.[31m9000[30m     [32m✔[30m [34mpurrr  [30m 0.2.4     
[32m✔[30m [34mtibble [30m 1.4.2          [32m✔[30m [34mdplyr  [30m 0.7.4     
[32m✔[30m [34mtidyr  [30m 0.8.0          [32m✔[30m [34mstringr[30m 1.2.0     
[32m✔[30m [34mreadr  [30m 1.1.1          [32m✔[30m [34mforcats[30m 0.2.0     [39m
package ‘tibble’ was built under R version 3.4.3package ‘tidyr’ was built under R version 3.4.3[30m── [1mConflicts[22m ────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
[31m✖[30m [34mdplyr[30m::[32mfilter()[30m masks [34mstats[30m::filter()
[31m✖[30m [34mdplyr[30m::[32mlag()[30m    masks [34mstats[30m::lag()[39m

library(lubridate)


Attaching package: ‘lubridate’

The following object is masked from ‘package:base’:

    date

library(tidytext)   # created at Stack Overflow by Julia Silge and David Robinson
library(scales)


Attaching package: ‘scales’

The following object is masked from ‘package:purrr’:

    discard

The following object is masked from ‘package:readr’:

    col_factor

library(viridis)

Loading required package: viridisLite

Attaching package: ‘viridis’

The following object is masked from ‘package:viridisLite’:

    viridis.map

The following object is masked from ‘package:scales’:

    viridis_pal

library(ggrepel)
library(ggridges)
sessionInfo()

R version 3.4.2 (2017-09-28)
Platform: x86_64-apple-darwin15.6.0 (64-bit)
Running under: macOS High Sierra 10.13.3

Matrix products: default
BLAS: /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib
LAPACK: /Library/Frameworks/R.framework/Versions/3.4/Resources/lib/libRlapack.dylib

locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
 [1] ggridges_0.4.1     ggrepel_0.7.0      viridis_0.4.1      viridisLite_0.3.0  scales_0.5.0      
 [6] tidytext_0.1.6     lubridate_1.7.1    forcats_0.2.0      stringr_1.2.0      dplyr_0.7.4       
[11] purrr_0.2.4        readr_1.1.1        tidyr_0.8.0        tibble_1.4.2       ggplot2_2.2.1.9000
[16] tidyverse_1.2.1   

loaded via a namespace (and not attached):
 [1] reshape2_1.4.3    haven_1.1.1       lattice_0.20-35   colorspace_1.3-2  SnowballC_0.5.1  
 [6] yaml_2.1.16       rlang_0.1.6       pillar_1.1.0      foreign_0.8-69    glue_1.2.0       
[11] modelr_0.1.1      readxl_1.0.0      bindrcpp_0.2      bindr_0.1         plyr_1.8.4       
[16] munsell_0.4.3     gtable_0.2.0      cellranger_1.1.0  rvest_0.3.2       psych_1.7.8      
[21] knitr_1.19        parallel_3.4.2    broom_0.4.3       tokenizers_0.1.4  Rcpp_0.12.15     
[26] jsonlite_1.5      gridExtra_2.3     mnormt_1.5-5      hms_0.4.1         stringi_1.1.6    
[31] grid_3.4.2        cli_1.0.0         tools_3.4.2       magrittr_1.5      lazyeval_0.2.1   
[36] janeaustenr_0.1.5 crayon_1.3.4      pkgconfig_2.0.1   Matrix_1.2-12     xml2_1.2.0       
[41] assertthat_0.2.0  httr_1.3.1        rstudioapi_0.7    R6_2.2.2          nlme_3.1-131     
[46] compiler_3.4.2

Sys.setenv(TZ="America/Los_Angeles")
# https://brandcolors.net/b/stackoverflow
stack_overflow_color <- "#f48024"

theme_set(theme_minimal(base_size=9, base_family="Source Sans Pro") +
            theme(plot.title = element_text(size=8, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.1, unit='cm')),
                  axis.title.x = element_text(size=8),
                  axis.title.y = element_text(size=8),
                  plot.subtitle = element_text(family="Source Sans Pro Semibold", color="#969696", size=6),
                  plot.caption = element_text(size=6, color="#969696"),
                  legend.text = element_text(size = 6),
                  legend.key.width = unit(0.25, unit='cm')))

2 Behavior for new submissions

Use data precomputed from this BigQuery:

#standardSQL
SELECT
  DATE_TRUNC(DATE(creation_date), YEAR) AS year,
  SUM(view_count_delta) AS total_delta 
FROM (
  SELECT
    id,
    creation_date,
    b.view_count - a.view_count AS view_count_delta
  FROM
    `fh-bigquery.stackoverflow_archive.201703_posts_questions` a
  LEFT JOIN (
    SELECT
      id,
      view_count
    FROM
      `fh-bigquery.stackoverflow_archive.201712_posts_questions` ) b
  USING
    (id) )
GROUP BY
  year
ORDER BY
  year ASC

Load in the precomputed data.

file_path <- "stack_overflow_delta.csv"
df_deltas <- read_csv(file_path) %>% mutate(perc = total_delta / sum(as.numeric(total_delta)))

Parsed with column specification:
cols(
  year = col_date(format = ""),
  total_delta = col_integer()
)

df_deltas

Overview of 2017 view counts on older posts.

plot <- ggplot(df_deltas %>% filter(year >= ymd('2009-01-01'), year <= ymd('2016-01-01')), aes(x=year, y=perc)) +
          geom_bar(alpha=0.9, stat="identity", fill=stack_overflow_color) +
          scale_x_date(date_breaks='1 year', date_labels='%Y', minor_breaks = NULL) +
          scale_y_continuous(labels=percent) +
          labs(title='Proportion of 2017 Views on Older Stack Overflow Questions by Year',
                subtitle='From March 13th, 2017 to December 3rd, 2017. Visualization Excludes Partial Years',
               x='Year Question Was Posted',
               y='% of All Views',
               caption = "Max Woolf — minimaxir.com"
              )
ggsave('so_overview.png', plot, width=4, height=2)

Data processed from this BigQuery: (NB: to download large datasets, save as a BigQuery table and export as a CSV, then download the CSV)

#standardSQL
WITH
  answers_ordered AS (
  SELECT
    id,
    creation_date,
    parent_id AS question_id,
    score,
    ROW_NUMBER() OVER (PARTITION BY parent_id ORDER BY creation_date ASC) AS time_rank,
    ROW_NUMBER() OVER (PARTITION BY parent_id ORDER BY score DESC) AS score_rank,
    COUNT(*) OVER (PARTITION BY parent_id) AS num_answers
  FROM
    `fh-bigquery.stackoverflow_archive.201712_posts_answers` )
    
SELECT
  id,
  title,
  tags,
  DATETIME(creation_date) AS creation_date,
  accepted_answer_id,
  view_count,
  score,
  num_answers,
  f_answer_id,
  DATETIME(f_creation_date) AS f_creation_date,
  f_score,
  f_score_rank,
  a_answer_id,
  DATETIME(a_creation_date) AS a_creation_date,
  a_score,
  a_time_rank,
  a_score_rank,
  TIMESTAMP_DIFF(f_creation_date, creation_date, SECOND) AS time_to_f,
  TIMESTAMP_DIFF(a_creation_date, creation_date, SECOND) AS time_to_a
FROM
  `fh-bigquery.stackoverflow_archive.201712_posts_questions` q
LEFT JOIN (
  SELECT
    id AS f_answer_id,
    creation_date AS f_creation_date,
    question_id AS f_question_id,
    score AS f_score,
    score_rank AS f_score_rank,
    num_answers
  FROM
    answers_ordered
  WHERE
    time_rank = 1 ) f
ON
  q.id = f.f_question_id
LEFT JOIN (
  SELECT
    id AS a_answer_id,
    creation_date AS a_creation_date,
    score AS a_score,
    time_rank AS a_time_rank,
    score_rank AS a_score_rank
  FROM
    answers_ordered ) a
ON
  q.accepted_answer_id = a.a_answer_id
WHERE
  creation_date >= '2017-01-01 00:00:00' AND creation_date < '2017-12-01 00:00:00'

file_path <- "~/Downloads/stack_overflow_2017.csv"
df <- read_csv(file_path, progress=FALSE)

Parsed with column specification:
cols(
  id = col_integer(),
  title = col_character(),
  tags = col_character(),
  creation_date = col_datetime(format = ""),
  accepted_answer_id = col_integer(),
  view_count = col_integer(),
  score = col_integer(),
  num_answers = col_integer(),
  f_answer_id = col_integer(),
  f_creation_date = col_datetime(format = ""),
  f_score = col_integer(),
  f_score_rank = col_integer(),
  a_answer_id = col_integer(),
  a_creation_date = col_datetime(format = ""),
  a_score = col_integer(),
  a_time_rank = col_integer(),
  a_score_rank = col_integer(),
  time_to_f = col_integer(),
  time_to_a = col_integer()
)

df %>% head()

Add columns relevant to the timing when the post was made. The raw data is in UTC, so it must be converted to Eastern.

df <- df %>% mutate(
  creation_date = with_tz(creation_date, "America/New_York"),
  hour_posted = hour(creation_date),
  weekday_posted = wday(creation_date, label=T, abbr=F),
  week_posted = floor_date(creation_date, '1 week'),
  month_posted = floor_date(creation_date, '1 month')
)

Add a few bespoke features:

df <- df %>% mutate(
  num_answers = ifelse(is.na(num_answers), 0, num_answers),
  f_answer_in = ifelse(!is.na(time_to_f) & time_to_f < 60*60*4, 1, 0),
  a_answer_in = ifelse(!is.na(time_to_a) & time_to_a < 60*60*4, 1, 0),
  num_tags = 1 + str_match_all(tags, '\\|') %>% lapply(length) %>% unlist(),
  is_answered = ifelse(!is.na(time_to_a), 1, 0)
)

3 Overview

For all questions asked:

3.1 Weekly

plot <- ggplot(df, aes(x=week_posted, y=..count..)) +
  geom_bar(fill = stack_overflow_color, alpha=0.9) +
  scale_x_datetime(date_breaks='2 months', date_labels='%b') +
  scale_y_continuous(labels=comma) +
  labs(title='New Stack Overflow Questions in 2017',
       x='Week Question was Posted',
       y='# of Questions Posted',
       caption = "Max Woolf — minimaxir.com")
ggsave('weekly_count.png', plot, width=4, height=2)

4 Facet by Tags

4.1 By # of Tags

df_tag_counts <- df %>%
  group_by(num_tags) %>%
  summarize(count = n(),
            perc_answered = sum(is_answered)/count) %>%
  ungroup() %>%
  mutate(perc_all = count/sum(count)) %>%
  arrange(num_tags)
df_tag_counts

plot <- ggplot(df_tag_counts, aes(x=factor(num_tags), y = perc_all, fill=perc_all)) +
  geom_bar(stat='identity') +
  geom_text(aes(label=percent(perc_all), color=perc_all), vjust=-0.25, family="Source Sans Pro Bold", size=2.5) +
  scale_color_viridis(option='inferno', limits=c(0,1), guide=F) +
  scale_fill_viridis(option='inferno', limits=c(0,1), guide=F) +
  scale_y_continuous(labels=percent) +
  labs(title='Breakdown of # of Tags in Stack Overflow Questions',
       subtitle='For Questions Asked From January 2017 to November 2017',
       x='# of Tags in Question',
       y='% of Stack Overflow Questions',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')))
ggsave('so_tag_breakdown.png', plot, width=4, height=2.5)

5 Top Tags

df_top_tags <- df %>%
  select(creation_date, tags, score, num_answers, is_answered) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  group_by(tag) %>%
  summarize(count = n(),
            med_score = median(score),
            med_answers = median(num_answers),
            perc_is_answered = sum(is_answered)/count) %>%
  arrange(desc(count)) %>%
  head(1000)
df_top_tags %>% head()

df_top_tags_month <- df %>%
  select(month_posted, tags) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  filter(tag %in% (df_top_tags %>% head(40) %>% pull(tag))) %>%
  group_by(month_posted, tag) %>%
  summarize(count=n()) %>%
  arrange(month_posted) %>%
  filter(month_posted >= date('2017-01-01'))
df_top_tags_month %>% head()

5.1 Monthly by Tags

plot <- ggplot(df_top_tags_month, aes(x=month_posted, y=count, fill=tag)) +
  geom_bar(alpha=0.9, stat="identity") +
  geom_smooth(se=F, method="lm", color="black", size=0.5) +
  scale_x_datetime(date_breaks='3 months', date_labels='%b') +
  scale_y_continuous(labels=comma) +
  facet_wrap(~ tag, nrow=10, ncol=4, scales="free_y") +
  labs(title='New Stack Overflow Questions for the Top 40 Tags',
       subtitle='From January 2017 to November 2017',
       x='Month Question Was Posted',
       y='# of Questions Posted With Tag',
       caption = "Max Woolf — minimaxir.com"
       ) +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')))
ggsave('monthly_count_tag.png', plot, width=6, height=8)

## Day/Hour of Week tags were posted

df_top_tags_hr_doy <- df %>%
  select(hour_posted, weekday_posted, tags) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  filter(tag %in% (df_top_tags %>% head(40) %>% pull(tag))) %>%
  group_by(hour_posted, weekday_posted, tag) %>%
  summarize(count=n()) %>%
  ungroup() %>%
  group_by(tag) %>%
  mutate(proportion=count/sum(count))
df_top_tags_hr_doy %>% head()

plot <- ggplot(df_top_tags_hr_doy, aes(x=hour_posted, y=fct_rev(weekday_posted), fill=proportion)) +
  geom_raster(stat="identity", interpolate=F) +
  geom_vline(xintercept=9, color="white", size=0.5, alpha=0.75) +
  geom_vline(xintercept=17, color="white", size=0.5, alpha=0.75) +
  scale_x_discrete() +
  scale_y_discrete() +
  scale_fill_viridis(option="inferno", labels=percent) +
  facet_wrap(~ tag, nrow=10, ncol=4) +
  labs(title='New Stack Overflow Questions for the Top 40 Tags, by Time Posted',
       subtitle='From January 2017 to November 2017, Vertical Lines Indicate 9 AM - 5 PM Eastern',
       x='Hour Question Was Posted (12 AM - 11 PM Eastern Time)',
       y='Day of Week Question Was Posted',
       fill='Proportion of All Questions\nPosted w/ Tag',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'top',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
        axis.text.y = element_text(size = 5),
        legend.text = element_text(size = 6),
        legend.title = element_text(size = 6),
        legend.key.width = unit(1, unit='cm'),
        legend.key.height = unit(0.25, unit='cm'),
        legend.margin = margin(c(0, 0, -0.5, 0), unit='cm'))
ggsave('monthly_count_hr_doy.png', plot, width=6, height=8)

5.2 Tag Distributions

df_top_tags_distribution <- df %>%
  select(month_posted, tags, view_count, num_answers, time_to_f, time_to_a) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  filter(tag %in% (df_top_tags %>% head(40) %>% pull(tag)), month_posted >= date('2017-01-01'))
df_tags_medians <- df_top_tags_distribution %>%
  group_by(tag) %>%
  summarize(med=median(view_count)) %>%
  arrange(desc(med))
df_top_tags_distribution <- df_top_tags_distribution %>%
  mutate(tag = fct_rev(factor(tag, levels=(df_tags_medians %>% pull(tag)))))
df_top_tags_distribution %>% head()

plot <- ggplot(df_top_tags_distribution, aes(x=view_count, y=tag, fill=tag)) +
          #geom_boxplot() +
          geom_density_ridges(scale = 5, size = 0.25, rel_min_height = 0.03) +
          scale_y_discrete() +
          scale_x_log10(labels=comma, limits=c(10,10^4), breaks=10^(1:4), minor_breaks=NULL) +
          labs(title='New Stack Overflow Questions for the Top 40 Tags',
                subtitle='From January 2017 to November 2017',
               x='# of Views on Post',
               y='# of Views on Post',
               caption = "Max Woolf — minimaxir.com") +
          theme(legend.position = 'none',
                plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
                axis.title.y = element_blank())
ggsave('views_tag_dist.png', plot, width=6, height=8)

5.3 Tag Wordcloud

df_top_tags_words <- df %>%
                select(tags, title) %>%
                unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
                filter(tag %in% (df_top_tags %>% head(20) %>% pull(tag))) %>%
                unnest_tokens(word, title, token="words") %>%
                filter(!(word %in% stop_words$word)) %>%
                group_by(tag, word) %>%
                summarize(count=n()) %>%
                ungroup() %>%
                group_by(tag) %>%
                mutate(max_norm = count/max(count)) %>%
                arrange(desc(count)) %>%
                top_n(20) %>%
                arrange(tag, desc(count))

Selecting by max_norm

df_top_tags_words %>% head(50)

Wordcloud trick in ggplot2 adapted from Mhairi McNeill’s blog post.

set.seed(123)   # For geom_text_repel
plot <- ggplot(df_top_tags_words, aes(x = 1, y = 1, size = max_norm, label = word, color=max_norm)) +
  geom_text_repel(segment.size = 0, force = 100, family="Roboto Condensed Bold") +
  scale_size(range = c(2, 4), guide = FALSE) +
  scale_color_viridis(end=0.8, discrete=F, option="inferno", guide = FALSE) +
  scale_y_continuous(breaks = NULL) +
  scale_x_continuous(breaks = NULL) +
  facet_wrap(~ tag, nrow=10, ncol=4) +
  labs(title='Wordcloud of Words in Titles of Stack Overflow Questions for the Top 20 Tags',
       subtitle='From January 2017 to November 2017',
       x='Month Question Was Posted',
       y='# of Questions Posted With Tag',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
        axis.text.x = element_blank(),
        axis.title.x = element_blank(),
        axis.text.y = element_blank(),
        axis.title.y = element_blank()
  )
ggsave('so_tag_wordcloud.png', plot, width=6, height=8)

5.4 Distribution of Times to an acceptable answer

df_tag_median <- df_top_tags_distribution %>%
  group_by(tag) %>%
  summarize(count=n(),
    med = median(time_to_a, na.rm=T)) %>%
  arrange(med) %>%
  mutate(tag = fct_rev(factor(tag, levels=tag)))
df_tag_median %>% head(40)

df_tag_median <- df_tag_median %>%
  arrange(desc(count)) %>%
  head(40) %>%
  arrange(med) %>%
  mutate(tag = fct_rev(factor(tag, levels=tag)))
plot <- ggplot(df_top_tags_distribution %>% filter(!is.na(time_to_a), tag %in% (df_tag_median %>%  na.omit() %>% pull(tag))) %>% mutate(tag = factor(tag, levels=(df_tag_median %>%  na.omit() %>% pull(tag)))), aes(x=time_to_a, fill=tag)) +
  geom_histogram() +
  geom_vline(data=df_tag_median %>% mutate(tag = fct_relevel(factor(tag), (df_tag_median %>%  na.omit() %>% pull(tag)))), aes(xintercept=med), linetype="dashed") +
  scale_x_continuous(limits = c(0,4*60*60), labels=0:4, breaks=seq(0,4*60*60, 60*60)) +
  #scale_x_log10(limits = c(1,10^3*60*60), labels=c(0, 10^(0:3)), breaks=c(0, 10^(0:3)*60*60)) +
  scale_y_continuous(labels=comma) +
  facet_wrap(~ tag, nrow=10, ncol=4, scales="free_y") +
  labs(title='Distribution of Time-To-Answer Stack Overflow Questions for the Top 40 Tags',
       subtitle='From January 2017 to November 2017, Sorted by Lowest Median Time to Accepted Answer',
       x='Median Time (Hours) Until Accepted Answer is Posted For Questions in Tag',
       y='# of Questions Posted With Tag',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')))

Unknown levels in `f`: 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1

ggsave('acceptable_answer_density.png', plot, width=6, height=8)

A Visual Look at Stack Overflow’s Question Tags

Max Woolf (@minimaxir)

2017-02-09