chunkR devel
This package allows to read large text
tables in chunks, using a fast C++ backend. Text files can be imported
as data frames (with automatic column type detection option) or
matrices. The program is designed to be simple and user-friendly.
chunkR is based on three basic functions: chunker, to create a connection to a text file; next_chunk() to read the next chunk, and get_table() to retrieve the corresponding data chunk.
These functions can be easily included in loops and other source code, using the return value of the next_chunk() function, that is TRUE when a new chunk is available and FALSE when the file was totally read, respectively. The get_table() function, returns an empty data frame/matrix when next_chunk() is FALSE. See the examples below.
library(devtools)
install_github("leandroroser/chunkR")data(iris)
# write iris as tab delimited file. Note that quote is set to FALSE
tmp_path <- file.path(tempdir(),"iris.txt")
write.table(iris, tmp_path, quote = FALSE)+#-----------------------------------------------------------------#
+#--- Reading a data frame with automatic column-type detection ---#
+#-----------------------------------------------------------------## create a 'chunker' object passing the path of the input file.
my_chunker_object <- chunker(tmp_path, chunksize = 30)
# read a chunk
next_chunk(my_chunker_object)
# get the chunk
get_table(my_chunker_object)
# read another chunk
next_chunk(my_chunker_object)
# get the number of lines already read
get_completed(my_chunker_object)-#---- Quoted data --------#write.table(iris, tmp_path, quote = TRUE)
my_chunker_object <- chunker(tmp_path, quoted = TRUE, chunksize = 30)
next_chunk(my_chunker_object)
get_table(my_chunker_object)-#---- Data without rownames and/or colnames ----#tmp_path <- file.path(tempdir(),"iris.txt")
write.table(iris, tmp_path, row.names = FALSE, col.names = FALSE)
my_chunker_object2 <- chunker(tmp_path, quoted = TRUE, chunksize = 30,
has_rownames = FALSE, has_colnames = FALSE)
next_chunk(my_chunker_object2)
get_table(my_chunker_object2) # automatic generation of rownames and/or colnames-#--- read a csv file ---#tmp_path_csv <- file.path(tempdir(),"iris.csv")
write.table(iris, tmp_path_csv, quote = FALSE, sep = ",")
# read the csv indicating the value of the sep parameter
my_chunker_object3 <- chunker(tmp_path_csv, chunksize = 30, sep = ",")
# the file can then be processed as with tab delimiters
next_chunk(my_chunker_object3)
get_table(my_chunker_object3)
# remove temporal file
file.remove(tmp_path_csv)+#--------------------------------------------------------#
+#--- Reading a data frame using column types argument ---#
+#--------------------------------------------------------### Four types can be passed : "character", "numeric" (aka "double"), "integer", "logical"
# create a 'chunker' object passing the path of the input file.
my_chunker_object4 <- chunker(tmp_path, chunksize = 120,
columns_classes = c("numeric", "numeric", "numeric","numeric", "character"))
# read a chunk
next_chunk(my_chunker_object4)
# get the chunk
get_table(my_chunker_object4)
# read another chunk
next_chunk(my_chunker_object4)
# get the number of lines already read
get_completed(my_chunker_object4)+#-------------------------#
+#--- Reading a matrix ---#
+#-------------------------#my_chunker_object5 <- chunker(tmp_path, chunksize = 30, data_format= "matrix")
# read a chunk
next_chunk(my_chunker_object5)
# store the chunk as a character matrix in R
this_data <- get_table(my_chunker_object5)
# The package provides a fast generic C++ function for conversion from
# matrix (any R type) to data frame
this_data_as_df2 <- matrix2df(this_data)
# remove temporal file
file.remove(tmp_path)+#----------------------------------#
+#--- Example with a big table -----#
+#----------------------------------#-### Example with a data frame# create a large data frame, and write it in a temporal directory
tmp_path <- file.path(tempdir(),"big_table.txt")
out <- data.frame(numeric_data = runif(1000000),
character_data = sample(c("a", "t", "c", "g"), 1000000,
replace = TRUE),
integer_data = sample(1000000),
bool_data = sample(c(TRUE, FALSE), 1000000, replace = TRUE))
write.table(out, tmp_path, quote = FALSE)
# create a chunker object, reading in chunks of 10000 lines
my_chunker_object6 <- chunker(tmp_path, chunksize = 10000)
next_chunk(my_chunker_object6)
data <- get_table(my_chunker_object6)
# check classes
lapply(data,typeof)
file.remove(tmp_path)-### Example with a matrix# create a large matrix, and write it in a temporal directory
my_table <- tempfile()
write.table(matrix(sample(c("a", "t", "c", "g"), 1000000, replace = TRUE),
100000, 1000), my_table, quote = FALSE)
# create a chunker object, reading in chunks of 10000 lines
my_chunker_object7 <- chunker(my_table, chunksize = 10000, data_format= "matrix")
# create a loop to read all the file and do something with it
lines <- 0
while(next_chunk(my_chunker_object7))
{
data <- get_table(my_chunker_object7)
# do something with data, e.g., convert to data frame first
data <- matrix2df(data)
lines <- lines + nrow(data)
cat("Processed ", lines, "lines\n")
}
# remove the temporal file
file.remove(my_table)