Added line to replace multiple whitespace characters in full text by a single regular whitespace

master
Erik de Vries 6 years ago
parent 0e45c0f2d1
commit dc4daf9de4

@ -4,7 +4,7 @@
#' @param out The elasticizer-generated data frame #' @param out The elasticizer-generated data frame
#' @param words String indicating the number of words to keep from each document (maximum document length), 999 indicates the whole document #' @param words String indicating the number of words to keep from each document (maximum document length), 999 indicates the whole document
#' @param text String indicating whether the "merged" field will contain the "full" text, or "lemmas" #' @param text String indicating whether the "merged" field will contain the "full" text, or "lemmas"
#' @return A Quanteda dfm #' @return A Quanteda dfm
#' @export #' @export
#' @examples #' @examples
#' dfm_gen(out, words = '999') #' dfm_gen(out, words = '999')
@ -32,7 +32,8 @@ dfm_gen <- function(out,words = '999', text = c("lemmas","full")) {
str_replace_na(out$`_source.text`, replacement = " "), str_replace_na(out$`_source.text`, replacement = " "),
sep = " ") %>% sep = " ") %>%
# Remove html tags # Remove html tags
str_replace_all("<.*?>", " ") str_replace_all("<.*?>", " ") %>%
str_replace_all("\\s+"," ")
} }
# out$codes <- out$`_source.codes.majorTopic` %>% # out$codes <- out$`_source.codes.majorTopic` %>%
out <- out %>% out <- out %>%
@ -58,4 +59,4 @@ dfm_gen <- function(out,words = '999', text = c("lemmas","full")) {
dfm <- corpus(out$merged, docnames = out$`_id`, docvars = out[,-seq(1,(length(names(out))-3),1)]) %>% dfm <- corpus(out$merged, docnames = out$`_id`, docvars = out[,-seq(1,(length(names(out))-3),1)]) %>%
dfm(tolower = T, stem = F, remove_punct = T, valuetype = "regex", ngrams = 1) dfm(tolower = T, stem = F, remove_punct = T, valuetype = "regex", ngrams = 1)
return(dfm) return(dfm)
} }

Loading…
Cancel
Save