c++实现解压+解包tar.gz文件
·
事情是这样的:
之前一直用libarchive来提取tar.gz文件,但是在qnx系统下,偶尔会发生解压失败的情况,不是每次都失败。后来分析了一下libarchive源码,发现libarchive在处理gz解压时(archive_read_support_filter_gzip)会fork子进程,然后调用linux的read函数时失败了,不知道具体原因。。。所以就想针对tar.gz写一个功能,具体需求具体实现,libarchive处理还是比较复杂的。(以下梳理编写来自M yaonan老哥)
实现逻辑还是比较简单的:
1.依赖zlib库,用来将xxx.tar.gz解压成xxx.tar。(使用gzopen,gzread)
2.根据tar format来extract出文件,tar format都是512 size的,具体可以google。


我的tar,gz 是USTAR的,可以通过二进制文本工具查看xxx.tar文件,会看到USTAR。下面是具体的结构的,注意的是tar文件最后都有1024个'\0'字符

tar存放的类型如下:
For USTAR, the typeflag field is a compatible extension of the link field of the older tar format. The following values are recognized:
Flag
File type
0 or null
Regular file
1
Link to another file already archived
2
Symbolic link
3
Character special file
4
Block special file (not supported)
5
Directory
6
FIFO special file
7
Reserved
S
z/OS extended USTAR special header
T
z/OS extended USTAR special header summary (S and T are z/OS extensions. See z/OS-extended USTAR support for more information.)
A-Z
Available for custom usage
// targz.h
#ifndef XXX_TARGZ_H
#define XXX_H
#include <string>
#include <vector>
namespace targz {
enum tarEntryType {
FileCompatibilityType = '\0',
FileType = '0',
HardlinkType = '1',
SymlinkType,
CharacterDeviceType,
BlockDeviceType,
DirType,
FifoType,
ReservedType,
OtherType
};
class tarGzFile {
public:
tarGzFile() {}
~tarGzFile() {}
void setExtractedPath(const std::string &path);
std::string getExtractedPath();
// decompress tar.gz file by zlib to tar buffer
bool decompressGzFile(const std::string &targz_file_path, std::vector<char> &tar_buffer);
// decompress tar.gz file by zlib to tar file
bool decompressGzFile(const std::string &targz_file_path);
// extract source file or dir from tar buffer
bool extractSource(const std::vector<char> &tar_buffer);
// extract source file or dir from tar file
bool extractSource();
private:
// The tar file ends on two Null headers
bool is_end_of_archive(const char *p);
// Check if the header is Null.
bool is_null_of_header(const char *p);
// Verify the checksum, actually, it stops the reading on error
int verifyChecksum(const char *p);
// Used to read the header
int parseoct(const char *p, std::size_t n);
void createFileStream(char *pathname, std::ofstream &f);
void wirteTarFile(char *pathname, const char *file_buff, int filesize);
// save decompress path
std::string savepath_ = "";
// substr xxx.tar.gz to xxx.tar
std::string tar_file_path_ = "";
};
} // namespace targz
// targz.cpp
#include "targz.h"
#include <dirent.h>
#include <stdio.h>
#include <zlib.h>
#include <fstream>
static const ushort TAR_FORMAT_CELL_SIZE = 512;
static const ushort TAR_END_TWO_NULL_HEADER_SIZE = 1024;
namespace targz {
static void createDir(char *pathname, int mode) {
char *p;
int r = -1;
/* Strip trailing '/' */
if (pathname[strlen(pathname) - 1] == '/') {
pathname[strlen(pathname) - 1] = '\0';
}
if (nullptr != opendir(pathname)) {
return;
}
/* Try creating the directory. */
#if defined(_WIN32) && !defined(__CYGWIN__)
r = _mkdir(pathname);
#else
r = mkdir(pathname, mode);
#endif
if (r != 0) {
/* On failure, try creating parent directory. */
p = strrchr(pathname, '/');
if (p != NULL) {
*p = '\0';
createDir(pathname, 0755);
*p = '/';
#if defined(_WIN32) && !defined(__CYGWIN__)
r = _mkdir(pathname);
#else
r = mkdir(pathname, mode);
#endif
}
}
if (r != 0) {
ERROR("[targz] create_dir failed :{}", pathname);
return;
}
INFO("[targz] create dir: {}", pathname);
}
void tarGzFile::createFileStream(char *pathname, std::ofstream &f) {
f.open(pathname, std::ios::out | std::ios::binary);
if (!f.is_open()) {
/* Try creating parent dir and then creating file. */
char *p = strrchr(pathname, '/');
if (p != NULL) {
*p = '\0';
createDir(pathname, 0755);
*p = '/';
f.open(pathname, std::ios::app | std::ios::binary);
if (!f.is_open()) {
ERROR("[targz] create file stream failed: {}", pathname);
return;
}
}
}
}
void tarGzFile::wirteTarFile(char *pathname, const char *file_buff, int filesize) {
INFO("[targz] write Tar File: {}", pathname);
std::ofstream f;
createFileStream(pathname, f);
if (f.is_open()) {
f.write(file_buff, filesize);
}
f.close();
}
bool tarGzFile::decompressGzFile(const std::string &targz_file_path, std::vector<char> &tar_buffer) {
tar_buffer.clear();
gzFile inFileZ = gzopen(targz_file_path.c_str(), "rb");
if (!inFileZ) {
ERROR("[targz]decompressGzFile open tar.gz failed: {}", targz_file_path);
return false;
}
char unzipBuffer[TAR_FORMAT_CELL_SIZE] = {0};
unsigned int unzippedBytes;
std::vector<char> unzippedData;
while (true) {
unzippedBytes = gzread(inFileZ, unzipBuffer, TAR_FORMAT_CELL_SIZE);
if (unzippedBytes > 0) {
tar_buffer.insert(tar_buffer.end(), unzipBuffer, unzipBuffer + unzippedBytes);
} else {
break;
}
}
gzclose(inFileZ);
return true;
}
bool tarGzFile::decompressGzFile(const std::string &targz_file_path) {
gzFile inFileZ = gzopen(targz_file_path.c_str(), "rb");
if (!inFileZ) {
ERROR("[targz]decompressGzFile open tar.gz failed: {}", targz_file_path);
return false;
}
auto find_index = targz_file_path.find_last_of(".");
tar_file_path_ = targz_file_path.substr(0, find_index);
std::ofstream writer(tar_file_path_.c_str(), std::ios::app | std::ios::binary);
if (!writer.is_open()) {
ERROR("[targz] decompressGzFile Open file {} error!", tar_file_path_);
return false;
}
char unzipBuffer[TAR_FORMAT_CELL_SIZE] = {0};
unsigned int unzippedBytes;
while (true) {
unzippedBytes = gzread(inFileZ, unzipBuffer, TAR_FORMAT_CELL_SIZE);
if (unzippedBytes > 0) {
writer.write(unzipBuffer, unzippedBytes);
} else {
break;
}
}
gzclose(inFileZ);
writer.close();
return true;
}
bool tarGzFile::extractSource(const std::vector<char> &tar_buffer) {
if (tar_buffer.size() < TAR_FORMAT_CELL_SIZE) {
ERROR("[targz] targz file size less than 512, format error!");
return false;
}
std::size_t bytes_read = 0;
std::vector<char> buff;
std::size_t tar_buffer_size = tar_buffer.size();
std::size_t vaild_buffer = tar_buffer_size - TAR_END_TWO_NULL_HEADER_SIZE;
buff.reserve(TAR_FORMAT_CELL_SIZE);
do {
buff.clear();
buff = {tar_buffer.begin() + bytes_read, tar_buffer.begin() + bytes_read + TAR_FORMAT_CELL_SIZE};
// read the header
if (is_null_of_header(&buff[0])) {
ERROR("[targz]extractSource read header all null!");
return false;
}
// Checksum verification
if (!verifyChecksum(&buff[0])) {
ERROR("[targz]extractSource checksum verification wrong!");
return false;
}
// read the filesize at buff[124], 12 bytes
int filesize = parseoct(&buff[0] + 124, 12);
// The actual size on disk of the file is a multiple of 512 in tar format
std::size_t nextEntry = TAR_FORMAT_CELL_SIZE;
if (filesize) { // filesize > 0
if (filesize < TAR_FORMAT_CELL_SIZE) {
nextEntry += TAR_FORMAT_CELL_SIZE;
} else if (filesize % TAR_FORMAT_CELL_SIZE == 0) {
nextEntry += filesize;
} else {
nextEntry += (TAR_FORMAT_CELL_SIZE * (1 + filesize / TAR_FORMAT_CELL_SIZE));
}
}
tarEntryType type = static_cast<tarEntryType>(buff[156]);
std::string filename(&buff[0], &buff[99]);
switch (type) {
case FileCompatibilityType:
case FileType: {
std::string tmp_path = getExtractedPath() + filename.c_str();
char tmpname[tmp_path.size() + 1] = {0};
strncpy(tmpname, tmp_path.c_str(), tmp_path.length());
wirteTarFile(tmpname, (&tar_buffer[0] + bytes_read + TAR_FORMAT_CELL_SIZE), filesize);
break;
}
case DirType: {
std::string tmp_path = getExtractedPath() + filename.c_str();
char dirname[tmp_path.size() + 1] = {0};
strncpy(dirname, tmp_path.c_str(), tmp_path.length());
createDir(dirname, parseoct(&buff[0] + 100, 8)); // File mode
break;
}
default:
WARN("[targz]extractSource default type warn:{}", (int)type);
break;
}
bytes_read += nextEntry;
} while (bytes_read < vaild_buffer);
if (bytes_read != vaild_buffer) {
ERROR("[targz]extractSource extract size wrong:{} vs {}", bytes_read, tar_buffer.size());
return false;
}
return true;
}
bool tarGzFile::extractSource() {
FILE *a = fopen(tar_file_path_.c_str(), "rb");
if (!a) {
ERROR("[targz] open tar file failed: {}", tar_file_path_);
return false;
}
bool res = false;
char buff[TAR_FORMAT_CELL_SIZE]{};
size_t bytes_read = 0;
int filesize = 0;
for (;;) {
std::ofstream writer;
bytes_read = fread(buff, 1, TAR_FORMAT_CELL_SIZE, a);
if (bytes_read < TAR_FORMAT_CELL_SIZE) {
ERROR("[targz] targz file size less than 512, format error!");
break;
}
if (is_end_of_archive(buff)) {
INFO("[targz] End tar gz file extract finish!");
res = true;
break;
}
if (!verifyChecksum(buff)) {
ERROR("[targz]extractSource checksum verification wrong!");
break;
}
std::string filename(&buff[0], &buff[99]);
filesize = parseoct(buff + 124, 12);
tarEntryType type = static_cast<tarEntryType>(buff[156]);
switch (type) {
case FileCompatibilityType:
case FileType: {
std::string tmp_path = getExtractedPath() + filename.c_str();
char tmpname[tmp_path.size() + 1] = {0};
strncpy(tmpname, tmp_path.c_str(), tmp_path.length());
createFileStream(tmpname, writer);
break;
}
case DirType: {
std::string tmp_path = getExtractedPath() + filename.c_str();
char dirname[tmp_path.size() + 1] = {0};
strncpy(dirname, tmp_path.c_str(), tmp_path.length());
createDir(dirname, parseoct(buff + 100, 8));
filesize = 0;
break;
}
default:
WARN("[targz]extractSource default type warn:{}", (int)type);
break;
}
while (filesize > 0) {
bytes_read = fread(buff, 1, TAR_FORMAT_CELL_SIZE, a);
if (bytes_read < TAR_FORMAT_CELL_SIZE) {
ERROR("[targz] extractSource write failed: {}", filename);
return false;
}
if (filesize < TAR_FORMAT_CELL_SIZE) {
bytes_read = filesize;
}
if (writer.is_open()) {
writer.write(buff, bytes_read);
}
else {
ERROR("[targz] extractSource write file failed: {}", filename);
break;
}
filesize -= bytes_read;
}
writer.close();
}
return res;
}
bool tarGzFile::is_end_of_archive(const char *p) {
int n;
for (n = 511; n >= 0; --n) {
if (p[n] != '\0')
return false;
}
return true;
}
bool tarGzFile::is_null_of_header(const char *p) {
int n;
for (n = 511; n >= 0; --n) {
if (p[n] != '\0')
return false;
}
return true;
}
/* Parse an octal number, ignoring leading and trailing nonsense. */
int tarGzFile::parseoct(const char *p, std::size_t n) {
int i = 0;
while ((*p < '0' || *p > '7') && n > 0) {
++p;
--n;
}
while (*p >= '0' && *p <= '7' && n > 0) {
i *= 8;
i += *p - '0';
++p;
--n;
}
return (i);
}
/* Verify the tar checksum. */
int tarGzFile::verifyChecksum(const char *p) {
int n, u = 0;
for (n = 0; n < 512; ++n) {
if (n < 148 || n > 155)
/* Standard tar checksum adds unsigned bytes. */
u += ((unsigned char *)p)[n];
else
u += 0x20;
}
return (u == parseoct(p + 148, 8));
}
void tarGzFile::setExtractedPath(const std::string &path) {
if (path.empty()) {
ERROR("[targz] not set path where extract to!");
return;
}
if (nullptr == opendir(path.c_str())) {
ERROR("[targz] extract path not exist: {}", path);
return;
}
savepath_ = path;
}
std::string tarGzFile::getExtractedPath() { return savepath_; }
} // namespace targz
// 测试
int main(int argc, char *argv[]) {
if (argc < 3) {
INFO("input param number missin,please input tar gz file path and save dir path(end add /)....");
return -1;
}
std::string source_filename = argv[1];
std::string save_dir = argv[2];
std::unique_ptr<targz::tarGzFile> handle_tmp = std::make_unique<targz::tarGzFile>();
handle_tmp->setExtractedPath(save_dir);
std::vector<char> tar_buff;
handle_tmp->decompressGzFile(source_filename, tar_buff);
handle_tmp->extractSource(tar_buff);
return 0;
}
魔乐社区(Modelers.cn) 是一个中立、公益的人工智能社区,提供人工智能工具、模型、数据的托管、展示与应用协同服务,为人工智能开发及爱好者搭建开放的学习交流平台。社区通过理事会方式运作,由全产业链共同建设、共同运营、共同享有,推动国产AI生态繁荣发展。
更多推荐


所有评论(0)