A Visual Look at Stack Overflow’s Question Tags

This R Notebook is the complement to my blog post A Visual Overview of Stack Overflow’s Question Tags.

This notebook is licensed under the MIT License. If you use the code or data visualization designs contained within this notebook, it would be greatly appreciated if proper attribution is given back to this notebook and/or myself. Thanks! :)

1 Setup

library(tidyverse)

── Attaching packages ─────────────────────────────────────────────────────────────────── tidyverse 1.2.1 ──
✔ ggplot2 2.2.1.9000     ✔ purrr   0.2.4     
✔ tibble  1.4.2          ✔ dplyr   0.7.4     
✔ tidyr   0.8.0          ✔ stringr 1.2.0     
✔ readr   1.1.1          ✔ forcats 0.2.0     
package ‘tibble’ was built under R version 3.4.3package ‘tidyr’ was built under R version 3.4.3── Conflicts ────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()

library(lubridate)

Attaching package: ‘lubridate’

The following object is masked from ‘package:base’:

    date

library(tidytext)   # created at Stack Overflow by Julia Silge and David Robinson
library(scales)

Attaching package: ‘scales’

The following object is masked from ‘package:purrr’:

    discard

The following object is masked from ‘package:readr’:

    col_factor

library(viridis)

Loading required package: viridisLite

Attaching package: ‘viridis’

The following object is masked from ‘package:viridisLite’:

    viridis.map

The following object is masked from ‘package:scales’:

    viridis_pal

library(ggrepel)
library(ggridges)
sessionInfo()

R version 3.4.2 (2017-09-28)
Platform: x86_64-apple-darwin15.6.0 (64-bit)
Running under: macOS High Sierra 10.13.3

Matrix products: default
BLAS: /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib
LAPACK: /Library/Frameworks/R.framework/Versions/3.4/Resources/lib/libRlapack.dylib

locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
 [1] ggridges_0.4.1     ggrepel_0.7.0      viridis_0.4.1      viridisLite_0.3.0  scales_0.5.0      
 [6] tidytext_0.1.6     lubridate_1.7.1    forcats_0.2.0      stringr_1.2.0      dplyr_0.7.4       
[11] purrr_0.2.4        readr_1.1.1        tidyr_0.8.0        tibble_1.4.2       ggplot2_2.2.1.9000
[16] tidyverse_1.2.1   

loaded via a namespace (and not attached):
 [1] reshape2_1.4.3    haven_1.1.1       lattice_0.20-35   colorspace_1.3-2  SnowballC_0.5.1  
 [6] yaml_2.1.16       rlang_0.1.6       pillar_1.1.0      foreign_0.8-69    glue_1.2.0       
[11] modelr_0.1.1      readxl_1.0.0      bindrcpp_0.2      bindr_0.1         plyr_1.8.4       
[16] munsell_0.4.3     gtable_0.2.0      cellranger_1.1.0  rvest_0.3.2       psych_1.7.8      
[21] knitr_1.19        parallel_3.4.2    broom_0.4.3       tokenizers_0.1.4  Rcpp_0.12.15     
[26] jsonlite_1.5      gridExtra_2.3     mnormt_1.5-5      hms_0.4.1         stringi_1.1.6    
[31] grid_3.4.2        cli_1.0.0         tools_3.4.2       magrittr_1.5      lazyeval_0.2.1   
[36] janeaustenr_0.1.5 crayon_1.3.4      pkgconfig_2.0.1   Matrix_1.2-12     xml2_1.2.0       
[41] assertthat_0.2.0  httr_1.3.1        rstudioapi_0.7    R6_2.2.2          nlme_3.1-131     
[46] compiler_3.4.2   

Sys.setenv(TZ="America/Los_Angeles")
# https://brandcolors.net/b/stackoverflow
stack_overflow_color <- "#f48024"

theme_set(theme_minimal(base_size=9, base_family="Source Sans Pro") +
            theme(plot.title = element_text(size=8, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.1, unit='cm')),
                  axis.title.x = element_text(size=8),
                  axis.title.y = element_text(size=8),
                  plot.subtitle = element_text(family="Source Sans Pro Semibold", color="#969696", size=6),
                  plot.caption = element_text(size=6, color="#969696"),
                  legend.text = element_text(size = 6),
                  legend.key.width = unit(0.25, unit='cm')))

2 Behavior for new submissions

Use data precomputed from this BigQuery:

#standardSQL
SELECT
  DATE_TRUNC(DATE(creation_date), YEAR) AS year,
  SUM(view_count_delta) AS total_delta 
FROM (
  SELECT
    id,
    creation_date,
    b.view_count - a.view_count AS view_count_delta
  FROM
    `fh-bigquery.stackoverflow_archive.201703_posts_questions` a
  LEFT JOIN (
    SELECT
      id,
      view_count
    FROM
      `fh-bigquery.stackoverflow_archive.201712_posts_questions` ) b
  USING
    (id) )
GROUP BY
  year
ORDER BY
  year ASC

Load in the precomputed data.

file_path <- "stack_overflow_delta.csv"
df_deltas <- read_csv(file_path) %>% mutate(perc = total_delta / sum(as.numeric(total_delta)))

Parsed with column specification:
cols(
  year = col_date(format = ""),
  total_delta = col_integer()
)

df_deltas

Overview of 2017 view counts on older posts.

plot <- ggplot(df_deltas %>% filter(year >= ymd('2009-01-01'), year <= ymd('2016-01-01')), aes(x=year, y=perc)) +
          geom_bar(alpha=0.9, stat="identity", fill=stack_overflow_color) +
          scale_x_date(date_breaks='1 year', date_labels='%Y', minor_breaks = NULL) +
          scale_y_continuous(labels=percent) +
          labs(title='Proportion of 2017 Views on Older Stack Overflow Questions by Year',
                subtitle='From March 13th, 2017 to December 3rd, 2017. Visualization Excludes Partial Years',
               x='Year Question Was Posted',
               y='% of All Views',
               caption = "Max Woolf — minimaxir.com"
              )
ggsave('so_overview.png', plot, width=4, height=2)

Data processed from this BigQuery: (NB: to download large datasets, save as a BigQuery table and export as a CSV, then download the CSV)

#standardSQL
WITH
  answers_ordered AS (
  SELECT
    id,
    creation_date,
    parent_id AS question_id,
    score,
    ROW_NUMBER() OVER (PARTITION BY parent_id ORDER BY creation_date ASC) AS time_rank,
    ROW_NUMBER() OVER (PARTITION BY parent_id ORDER BY score DESC) AS score_rank,
    COUNT(*) OVER (PARTITION BY parent_id) AS num_answers
  FROM
    `fh-bigquery.stackoverflow_archive.201712_posts_answers` )
    
SELECT
  id,
  title,
  tags,
  DATETIME(creation_date) AS creation_date,
  accepted_answer_id,
  view_count,
  score,
  num_answers,
  f_answer_id,
  DATETIME(f_creation_date) AS f_creation_date,
  f_score,
  f_score_rank,
  a_answer_id,
  DATETIME(a_creation_date) AS a_creation_date,
  a_score,
  a_time_rank,
  a_score_rank,
  TIMESTAMP_DIFF(f_creation_date, creation_date, SECOND) AS time_to_f,
  TIMESTAMP_DIFF(a_creation_date, creation_date, SECOND) AS time_to_a
FROM
  `fh-bigquery.stackoverflow_archive.201712_posts_questions` q
LEFT JOIN (
  SELECT
    id AS f_answer_id,
    creation_date AS f_creation_date,
    question_id AS f_question_id,
    score AS f_score,
    score_rank AS f_score_rank,
    num_answers
  FROM
    answers_ordered
  WHERE
    time_rank = 1 ) f
ON
  q.id = f.f_question_id
LEFT JOIN (
  SELECT
    id AS a_answer_id,
    creation_date AS a_creation_date,
    score AS a_score,
    time_rank AS a_time_rank,
    score_rank AS a_score_rank
  FROM
    answers_ordered ) a
ON
  q.accepted_answer_id = a.a_answer_id
WHERE
  creation_date >= '2017-01-01 00:00:00' AND creation_date < '2017-12-01 00:00:00'

file_path <- "~/Downloads/stack_overflow_2017.csv"
df <- read_csv(file_path, progress=FALSE)

Parsed with column specification:
cols(
  id = col_integer(),
  title = col_character(),
  tags = col_character(),
  creation_date = col_datetime(format = ""),
  accepted_answer_id = col_integer(),
  view_count = col_integer(),
  score = col_integer(),
  num_answers = col_integer(),
  f_answer_id = col_integer(),
  f_creation_date = col_datetime(format = ""),
  f_score = col_integer(),
  f_score_rank = col_integer(),
  a_answer_id = col_integer(),
  a_creation_date = col_datetime(format = ""),
  a_score = col_integer(),
  a_time_rank = col_integer(),
  a_score_rank = col_integer(),
  time_to_f = col_integer(),
  time_to_a = col_integer()
)

df %>% head()

Add columns relevant to the timing when the post was made. The raw data is in UTC, so it must be converted to Eastern.

df <- df %>% mutate(
  creation_date = with_tz(creation_date, "America/New_York"),
  hour_posted = hour(creation_date),
  weekday_posted = wday(creation_date, label=T, abbr=F),
  week_posted = floor_date(creation_date, '1 week'),
  month_posted = floor_date(creation_date, '1 month')
)

Add a few bespoke features:

df <- df %>% mutate(
  num_answers = ifelse(is.na(num_answers), 0, num_answers),
  f_answer_in = ifelse(!is.na(time_to_f) & time_to_f < 60*60*4, 1, 0),
  a_answer_in = ifelse(!is.na(time_to_a) & time_to_a < 60*60*4, 1, 0),
  num_tags = 1 + str_match_all(tags, '\\|') %>% lapply(length) %>% unlist(),
  is_answered = ifelse(!is.na(time_to_a), 1, 0)
)

3 Overview

For all questions asked:

3.1 Weekly

plot <- ggplot(df, aes(x=week_posted, y=..count..)) +
  geom_bar(fill = stack_overflow_color, alpha=0.9) +
  scale_x_datetime(date_breaks='2 months', date_labels='%b') +
  scale_y_continuous(labels=comma) +
  labs(title='New Stack Overflow Questions in 2017',
       x='Week Question was Posted',
       y='# of Questions Posted',
       caption = "Max Woolf — minimaxir.com")
ggsave('weekly_count.png', plot, width=4, height=2)

4 Facet by Tags

4.1 By # of Tags

df_tag_counts <- df %>%
  group_by(num_tags) %>%
  summarize(count = n(),
            perc_answered = sum(is_answered)/count) %>%
  ungroup() %>%
  mutate(perc_all = count/sum(count)) %>%
  arrange(num_tags)
df_tag_counts

plot <- ggplot(df_tag_counts, aes(x=factor(num_tags), y = perc_all, fill=perc_all)) +
  geom_bar(stat='identity') +
  geom_text(aes(label=percent(perc_all), color=perc_all), vjust=-0.25, family="Source Sans Pro Bold", size=2.5) +
  scale_color_viridis(option='inferno', limits=c(0,1), guide=F) +
  scale_fill_viridis(option='inferno', limits=c(0,1), guide=F) +
  scale_y_continuous(labels=percent) +
  labs(title='Breakdown of # of Tags in Stack Overflow Questions',
       subtitle='For Questions Asked From January 2017 to November 2017',
       x='# of Tags in Question',
       y='% of Stack Overflow Questions',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')))
ggsave('so_tag_breakdown.png', plot, width=4, height=2.5)

5 Top Tags

df_top_tags <- df %>%
  select(creation_date, tags, score, num_answers, is_answered) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  group_by(tag) %>%
  summarize(count = n(),
            med_score = median(score),
            med_answers = median(num_answers),
            perc_is_answered = sum(is_answered)/count) %>%
  arrange(desc(count)) %>%
  head(1000)
df_top_tags %>% head()               

df_top_tags_month <- df %>%
  select(month_posted, tags) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  filter(tag %in% (df_top_tags %>% head(40) %>% pull(tag))) %>%
  group_by(month_posted, tag) %>%
  summarize(count=n()) %>%
  arrange(month_posted) %>%
  filter(month_posted >= date('2017-01-01'))
df_top_tags_month %>% head()               

5.1 Monthly by Tags

plot <- ggplot(df_top_tags_month, aes(x=month_posted, y=count, fill=tag)) +
  geom_bar(alpha=0.9, stat="identity") +
  geom_smooth(se=F, method="lm", color="black", size=0.5) +
  scale_x_datetime(date_breaks='3 months', date_labels='%b') +
  scale_y_continuous(labels=comma) +
  facet_wrap(~ tag, nrow=10, ncol=4, scales="free_y") +
  labs(title='New Stack Overflow Questions for the Top 40 Tags',
       subtitle='From January 2017 to November 2017',
       x='Month Question Was Posted',
       y='# of Questions Posted With Tag',
       caption = "Max Woolf — minimaxir.com"
       ) +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')))
ggsave('monthly_count_tag.png', plot, width=6, height=8)

## Day/Hour of Week tags were posted

Related: https://stackoverflow.blog/2017/04/19/programming-languages-used-late-night/

df_top_tags_hr_doy <- df %>%
  select(hour_posted, weekday_posted, tags) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  filter(tag %in% (df_top_tags %>% head(40) %>% pull(tag))) %>%
  group_by(hour_posted, weekday_posted, tag) %>%
  summarize(count=n()) %>%
  ungroup() %>%
  group_by(tag) %>%
  mutate(proportion=count/sum(count))
df_top_tags_hr_doy %>% head() 

plot <- ggplot(df_top_tags_hr_doy, aes(x=hour_posted, y=fct_rev(weekday_posted), fill=proportion)) +
  geom_raster(stat="identity", interpolate=F) +
  geom_vline(xintercept=9, color="white", size=0.5, alpha=0.75) +
  geom_vline(xintercept=17, color="white", size=0.5, alpha=0.75) +
  scale_x_discrete() +
  scale_y_discrete() +
  scale_fill_viridis(option="inferno", labels=percent) +
  facet_wrap(~ tag, nrow=10, ncol=4) +
  labs(title='New Stack Overflow Questions for the Top 40 Tags, by Time Posted',
       subtitle='From January 2017 to November 2017, Vertical Lines Indicate 9 AM - 5 PM Eastern',
       x='Hour Question Was Posted (12 AM - 11 PM Eastern Time)',
       y='Day of Week Question Was Posted',
       fill='Proportion of All Questions\nPosted w/ Tag',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'top',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
        axis.text.y = element_text(size = 5),
        legend.text = element_text(size = 6),
        legend.title = element_text(size = 6),
        legend.key.width = unit(1, unit='cm'),
        legend.key.height = unit(0.25, unit='cm'),
        legend.margin = margin(c(0, 0, -0.5, 0), unit='cm'))
ggsave('monthly_count_hr_doy.png', plot, width=6, height=8)

5.2 Tag Distributions

df_top_tags_distribution <- df %>%
  select(month_posted, tags, view_count, num_answers, time_to_f, time_to_a) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  filter(tag %in% (df_top_tags %>% head(40) %>% pull(tag)), month_posted >= date('2017-01-01'))
df_tags_medians <- df_top_tags_distribution %>%
  group_by(tag) %>%
  summarize(med=median(view_count)) %>%
  arrange(desc(med))
df_top_tags_distribution <- df_top_tags_distribution %>%
  mutate(tag = fct_rev(factor(tag, levels=(df_tags_medians %>% pull(tag)))))
df_top_tags_distribution %>% head()

plot <- ggplot(df_top_tags_distribution, aes(x=view_count, y=tag, fill=tag)) +
          #geom_boxplot() +
          geom_density_ridges(scale = 5, size = 0.25, rel_min_height = 0.03) +
          scale_y_discrete() +
          scale_x_log10(labels=comma, limits=c(10,10^4), breaks=10^(1:4), minor_breaks=NULL) +
          labs(title='New Stack Overflow Questions for the Top 40 Tags',
                subtitle='From January 2017 to November 2017',
               x='# of Views on Post',
               y='# of Views on Post',
               caption = "Max Woolf — minimaxir.com") +
          theme(legend.position = 'none',
                plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
                axis.title.y = element_blank())
ggsave('views_tag_dist.png', plot, width=6, height=8)

5.3 Tag Wordcloud

df_top_tags_words <- df %>%
                select(tags, title) %>%
                unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
                filter(tag %in% (df_top_tags %>% head(20) %>% pull(tag))) %>%
                unnest_tokens(word, title, token="words") %>%
                filter(!(word %in% stop_words$word)) %>%
                group_by(tag, word) %>%
                summarize(count=n()) %>%
                ungroup() %>%
                group_by(tag) %>%
                mutate(max_norm = count/max(count)) %>%
                arrange(desc(count)) %>%
                top_n(20) %>%
                arrange(tag, desc(count))

Selecting by max_norm

df_top_tags_words %>% head(50) 

Wordcloud trick in ggplot2 adapted from Mhairi McNeill’s blog post.

set.seed(123)   # For geom_text_repel
plot <- ggplot(df_top_tags_words, aes(x = 1, y = 1, size = max_norm, label = word, color=max_norm)) +
  geom_text_repel(segment.size = 0, force = 100, family="Roboto Condensed Bold") +
  scale_size(range = c(2, 4), guide = FALSE) +
  scale_color_viridis(end=0.8, discrete=F, option="inferno", guide = FALSE) +
  scale_y_continuous(breaks = NULL) +
  scale_x_continuous(breaks = NULL) +
  facet_wrap(~ tag, nrow=10, ncol=4) +
  labs(title='Wordcloud of Words in Titles of Stack Overflow Questions for the Top 20 Tags',
       subtitle='From January 2017 to November 2017',
       x='Month Question Was Posted',
       y='# of Questions Posted With Tag',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
        axis.text.x = element_blank(),
        axis.title.x = element_blank(),
        axis.text.y = element_blank(),
        axis.title.y = element_blank()
  )
ggsave('so_tag_wordcloud.png', plot, width=6, height=8)

5.4 Distribution of Times to an acceptable answer

df_tag_median <- df_top_tags_distribution %>%
  group_by(tag) %>%
  summarize(count=n(),
    med = median(time_to_a, na.rm=T)) %>%
  arrange(med) %>%
  mutate(tag = fct_rev(factor(tag, levels=tag)))
df_tag_median %>% head(40)

df_tag_median <- df_tag_median %>%
  arrange(desc(count)) %>%
  head(40) %>%
  arrange(med) %>%
  mutate(tag = fct_rev(factor(tag, levels=tag)))
plot <- ggplot(df_top_tags_distribution %>% filter(!is.na(time_to_a), tag %in% (df_tag_median %>%  na.omit() %>% pull(tag))) %>% mutate(tag = factor(tag, levels=(df_tag_median %>%  na.omit() %>% pull(tag)))), aes(x=time_to_a, fill=tag)) +
  geom_histogram() +
  geom_vline(data=df_tag_median %>% mutate(tag = fct_relevel(factor(tag), (df_tag_median %>%  na.omit() %>% pull(tag)))), aes(xintercept=med), linetype="dashed") +
  scale_x_continuous(limits = c(0,4*60*60), labels=0:4, breaks=seq(0,4*60*60, 60*60)) +
  #scale_x_log10(limits = c(1,10^3*60*60), labels=c(0, 10^(0:3)), breaks=c(0, 10^(0:3)*60*60)) +
  scale_y_continuous(labels=comma) +
  facet_wrap(~ tag, nrow=10, ncol=4, scales="free_y") +
  labs(title='Distribution of Time-To-Answer Stack Overflow Questions for the Top 40 Tags',
       subtitle='From January 2017 to November 2017, Sorted by Lowest Median Time to Accepted Answer',
       x='Median Time (Hours) Until Accepted Answer is Posted For Questions in Tag',
       y='# of Questions Posted With Tag',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')))

Unknown levels in `f`: 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1

ggsave('acceptable_answer_density.png', plot, width=6, height=8)

df_top_tags_distribution <- df %>%
  select(tags, view_count, time_to_f, time_to_a, is_answered) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  filter(tag %in% (df_top_tags %>% pull(tag)))
df_top_tags_distribution %>% head()

5.4.1 Best/Worst Languages

This doesn’t necessairly imply one tool is “better” than another, the difference may be due to question difficulty and the number of people skilled in the technology.

df_acceptable_percs <- df_top_tags_distribution %>%
  group_by(tag) %>%
  summarize(count=n(),
            med_time_to_f = median(time_to_f, na.rm=T),
            med_time_to_a = median(time_to_a, na.rm=T),
            perc_is_answered = sum(is_answered) / count
  ) %>%
  arrange(desc(perc_is_answered))
df_acceptable_percs %>% head()

df_acceptable_percs_subset <- df_acceptable_percs %>%
  head(30) %>%
  mutate(tag = fct_rev(as_factor(tag)))
plot <- ggplot(df_acceptable_percs_subset, aes(x=tag, y = perc_is_answered, fill=perc_is_answered)) +
  geom_bar(stat='identity') +
  geom_text(aes(label=percent(perc_is_answered), color=perc_is_answered), hjust=-0.25, family="Source Sans Pro Bold", size=3.5) +
  coord_flip() +
  scale_color_viridis(option='inferno', limits=c(0,1), guide=F) +
  scale_fill_viridis(option='inferno', limits=c(0,1), guide=F) +
  scale_y_continuous(labels=percent, limits=c(0,1)) +
  labs(title='Top Tags on Stack Overflow for Questions w/ Accepted Answers',
       subtitle='For Questions Asked From January 2017 to November 2017, out of Top 1,000 Tags',
       x='Stack Overflow Question Tag',
       y='% of Tagged Questions Which Have an Accepted Answer',
       caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
        axis.title.y=element_blank())
ggsave('acceptable_answer_top_30.png', plot, width=6, height=6)

df_acceptable_percs_subset <- df_acceptable_percs %>%
  tail(30) %>%
  mutate(tag = fct_rev(as_factor(tag)))
plot <- ggplot(df_acceptable_percs_subset, aes(x=tag, y = perc_is_answered, fill=perc_is_answered)) +
  geom_bar(stat='identity') +
  geom_text(aes(label=percent(perc_is_answered), color=perc_is_answered), hjust=-0.25, family="Source Sans Pro Bold", size=3.5) +
  coord_flip() +
  scale_color_viridis(option='inferno', limits=c(0,1), guide=F) +
  scale_fill_viridis(option='inferno', limits=c(0,1), guide=F) +
  scale_y_continuous(labels=percent, limits=c(0,1)) +
  labs(title='Bottom Tags on Stack Overflow for Questions w/ Accepted Answers',
       subtitle='For Questions Asked From January 2017 to November 2017, out of Top 1,000 Tags',
x='Stack Overflow Question Tag',
y='% of Tagged Questions Which Have an Accepted Answer',
caption = "Max Woolf — minimaxir.com") +
  theme(legend.position = 'none',
        plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
        axis.title.y=element_blank())
ggsave('acceptable_answer_bottom_30.png', plot, width=6, height=6)

6 Adjacency Matrix for Top Tags

df_tag_adjacency <- df %>%
  select(id, tags, time_to_a, is_answered) %>%
  unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
  filter(tag %in% (df_top_tags %>% head(40) %>% pull(tag))) %>%
  inner_join(
    df %>%
      select(id, tags) %>%
      unnest_tokens(tag, tags, token=stringr::str_split, pattern=fixed("|")) %>%
      filter(tag %in% (df_top_tags %>% head(40) %>% pull(tag))), by=c("id" = "id")) %>%
  filter(tag.x != tag.y) %>%
  group_by(tag.x, tag.y) %>%
  summarize(count=n(),
            total_answered = sum(is_answered),
            perc_answered = total_answered/count) %>%
  arrange(tag.x, desc(count))

plot <- ggplot(df_tag_adjacency %>% filter(count >= 1000), aes(x=tag.x, y = tag.y, fill=count)) +
  geom_raster() +
  scale_fill_viridis(option='inferno', trans='log10', breaks=10^(1:5), labels=comma) +
  labs(title='Question Counts for Stack Overflow Tag Pairs',
       subtitle='For Questions Asked From January 2017 to November 2017 (min 1,000 questions per pair)',
       x='Stack Overflow Question Tag',
       fill='# of Questions',
       caption = "Max Woolf — minimaxir.com") +
  theme(
    axis.title.y=element_blank(),
    axis.title.x=element_blank(),
    axis.text.x = element_text(angle = 270, hjust = 0, vjust = 0.5),
    legend.position = 'top',
    plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
    #axis.text.y = element_text(size = 5),
    legend.text = element_text(size = 6),
    legend.title = element_text(size = 6),
    legend.key.width = unit(1, unit='cm'),
    legend.key.height = unit(0.25, unit='cm'),
    legend.margin = margin(c(0, 0, -0.1, 0), unit='cm'))
ggsave('so_tag_adjacency.png', plot, width=6, height=6)

plot <- ggplot(df_tag_adjacency %>% filter(count >= 1000), aes(x=tag.x, y = tag.y, fill=perc_answered)) +
  geom_raster() +
  geom_text(aes(label=sprintf("%0.0f", perc_answered*100)), family="Roboto Condensed Bold", size=2, color="white") +
  scale_fill_viridis(option='inferno', limits=c(0,1), breaks=pretty_breaks(4), labels=percent) +
  labs(title='Question Answer Rate for Stack Overflow Tag Pairs',
       subtitle='For Questions Asked From January 2017 to November 2017 (min 1,000 questions per pair)',
       x='Stack Overflow Question Tag',
       fill='% of Tagged Questions\nWhich Have an Accepted Answer',
       caption = "Max Woolf — minimaxir.com") +
  theme(
    axis.title.y=element_blank(),
    axis.title.x=element_blank(),
    axis.text.x = element_text(angle = 270, hjust=0, vjust = 0.5),
    legend.position = 'top',
    plot.title = element_text(size=10, family="Source Sans Pro Bold", margin=margin(t = -0.1, b = 0.0, unit='cm')),
    #axis.text.y = element_text(size = 5),
    legend.text = element_text(size = 6),
    legend.title = element_text(size = 6),
    legend.key.width = unit(1, unit='cm'),
    legend.key.height = unit(0.25, unit='cm'),
    legend.margin = margin(c(0, 0, -0.1, 0), unit='cm'))
ggsave('so_tag_adjacency_percent.png', plot, width=6, height=6)

7 LICENSE

The MIT License (MIT)

Copyright (c) 2018 Max Woolf

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.