@inproceedings{2eb32d0f367b4a14a305b5f36f874de6,
title = "High performance merging of massive data from genome-wide association studies",
abstract = "The traditional data processing methods working on single computer show less scalability and efficiency for performing ordered full-outer-joining, on merging large number of individual Genome-Wide Associations Studies (GWAS) data. Although the emerging of big data platforms such as Hadoop and Spark shed lights on this problem, the inefficiency of keeping data in total-sorted order as well as the workload imbalance problem limit their performance. In this study, we designed and compared three new methodologies based on MapReduce, HBase and Spark respectively, to merge hundreds of individuals VCF files on their Single Nucleotide Polymorphism (SNP) location into a single TPED file. Our methodologies overcame the limitations stated above and considerably improved the performance with good scalability on input size and computing resources.",
keywords = "Genome-Wide Association Studies (GWAS), HBase, MapReduce, Scalability, Spark, Total order full-outer-merging, TPED, Variant Call Format (VCF)",
author = "Xiaobo Sun and Fusheng Wang and Zhaohui Qin",
note = "Publisher Copyright: {\textcopyright} 2017, Springer International Publishing AG.; 3rd International Workshop on Data Management and Analytics for Medicine and Healthcare, DMAH 2017 held in conjunction with the 43rd International Conference on Very Large Data Bases, VLDB 2017 ; Conference date: 01-09-2017 Through 01-09-2017",
year = "2017",
doi = "10.1007/978-3-319-67186-4\_4",
language = "English",
isbn = "9783319671857",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Verlag",
pages = "36--40",
editor = "Edmon Begoli and Gang Luo and Fusheng Wang",
booktitle = "Data Management and Analytics for Medicine and Healthcare - 3rd International Workshop, DMAH 2017 Held at VLDB 2017, Proceedings",
}