/* * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. */ #if defined (__cplusplus) extern "C" { #endif /*-**************************************** * Dependencies ******************************************/ #include "util.h" /* note : ensure that platform.h is included first ! */ #include /* malloc, realloc, free */ #include /* fprintf */ #include /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ #include #include #if defined(_WIN32) # include /* utime */ # include /* _chmod */ #else # include /* chown, stat */ # if PLATFORM_POSIX_VERSION < 200809L || !defined(st_mtime) # include /* utime */ # else # include /* AT_FDCWD */ # include /* utimensat */ # endif #endif #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) #include /* needed for _mkdir in windows */ #endif #if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # include /* opendir, readdir */ # include /* strerror, memcpy */ #endif /* #ifdef _WIN32 */ /*-**************************************** * Internal Macros ******************************************/ /* CONTROL is almost like an assert(), but is never disabled. * It's designed for failures that may happen rarely, * but we don't want to maintain a specific error code path for them, * such as a malloc() returning NULL for example. * Since it's always active, this macro can trigger side effects. */ #define CONTROL(c) { \ if (!(c)) { \ UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \ __FILE__, __LINE__, #c); \ exit(1); \ } } /* console log */ #define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } static int g_traceDepth = 0; int g_traceFileStat = 0; #define UTIL_TRACE_CALL(...) \ { \ if (g_traceFileStat) { \ UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \ UTIL_DISPLAY(__VA_ARGS__); \ UTIL_DISPLAY("\n"); \ ++g_traceDepth; \ } \ } #define UTIL_TRACE_RET(ret) \ { \ if (g_traceFileStat) { \ --g_traceDepth; \ UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \ } \ } /* A modified version of realloc(). * If UTIL_realloc() fails the original block is freed. */ UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) { void *newptr = realloc(ptr, size); if (newptr) return newptr; free(ptr); return NULL; } #if defined(_MSC_VER) #define chmod _chmod #endif #ifndef ZSTD_HAVE_FCHMOD #if PLATFORM_POSIX_VERSION >= 199309L #define ZSTD_HAVE_FCHMOD #endif #endif #ifndef ZSTD_HAVE_FCHOWN #if PLATFORM_POSIX_VERSION >= 200809L #define ZSTD_HAVE_FCHOWN #endif #endif /*-**************************************** * Console log ******************************************/ int g_utilDisplayLevel; int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const char* acceptableLetters, int hasStdinInput) { int ch, result; if (hasStdinInput) { UTIL_DISPLAY("stdin is an input - not proceeding.\n"); return 1; } UTIL_DISPLAY("%s", prompt); ch = getchar(); result = 0; if (strchr(acceptableLetters, ch) == NULL) { UTIL_DISPLAY("%s \n", abortMsg); result = 1; } /* flush the rest */ while ((ch!=EOF) && (ch!='\n')) ch = getchar(); return result; } /*-************************************* * Constants ***************************************/ #define LIST_SIZE_INCREASE (8*1024) #define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50 /*-************************************* * Functions ***************************************/ void UTIL_traceFileStat(void) { g_traceFileStat = 1; } int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf) { int ret; UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename); #if defined(_MSC_VER) if (fd >= 0) { ret = !_fstat64(fd, statbuf); } else { ret = !_stat64(filename, statbuf); } #elif defined(__MINGW32__) && defined (__MSVCRT__) if (fd >= 0) { ret = !_fstati64(fd, statbuf); } else { ret = !_stati64(filename, statbuf); } #else if (fd >= 0) { ret = !fstat(fd, statbuf); } else { ret = !stat(filename, statbuf); } #endif UTIL_TRACE_RET(ret); return ret; } int UTIL_stat(const char* filename, stat_t* statbuf) { return UTIL_fstat(-1, filename, statbuf); } int UTIL_isRegularFile(const char* infilename) { stat_t statbuf; int ret; UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename); ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf); UTIL_TRACE_RET(ret); return ret; } int UTIL_isRegularFileStat(const stat_t* statbuf) { #if defined(_MSC_VER) return (statbuf->st_mode & S_IFREG) != 0; #else return S_ISREG(statbuf->st_mode) != 0; #endif } /* like chmod, but avoid changing permission of /dev/null */ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) { return UTIL_fchmod(-1, filename, statbuf, permissions); } int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions) { stat_t localStatBuf; UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions); if (statbuf == NULL) { if (!UTIL_fstat(fd, filename, &localStatBuf)) { UTIL_TRACE_RET(0); return 0; } statbuf = &localStatBuf; } if (!UTIL_isRegularFileStat(statbuf)) { UTIL_TRACE_RET(0); return 0; /* pretend success, but don't change anything */ } #ifdef ZSTD_HAVE_FCHMOD if (fd >= 0) { int ret; UTIL_TRACE_CALL("fchmod"); ret = fchmod(fd, permissions); UTIL_TRACE_RET(ret); UTIL_TRACE_RET(ret); return ret; } else #endif { int ret; UTIL_TRACE_CALL("chmod"); ret = chmod(filename, permissions); UTIL_TRACE_RET(ret); UTIL_TRACE_RET(ret); return ret; } } /* set access and modification times */ int UTIL_utime(const char* filename, const stat_t *statbuf) { int ret; UTIL_TRACE_CALL("UTIL_utime(%s)", filename); /* We check that st_mtime is a macro here in order to give us confidence * that struct stat has a struct timespec st_mtim member. We need this * check because there are some platforms that claim to be POSIX 2008 * compliant but which do not have st_mtim... */ #if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime) { /* (atime, mtime) */ struct timespec timebuf[2] = { {0, UTIME_NOW} }; timebuf[1] = statbuf->st_mtim; ret = utimensat(AT_FDCWD, filename, timebuf, 0); } #else { struct utimbuf timebuf; timebuf.actime = time(NULL); timebuf.modtime = statbuf->st_mtime; ret = utime(filename, &timebuf); } #endif errno = 0; UTIL_TRACE_RET(ret); return ret; } int UTIL_setFileStat(const char *filename, const stat_t *statbuf) { return UTIL_setFDStat(-1, filename, statbuf); } int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf) { int res = 0; stat_t curStatBuf; UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename); if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) { UTIL_TRACE_RET(-1); return -1; } /* Mimic gzip's behavior: * * "Change the group first, then the permissions, then the owner. * That way, the permissions will be correct on systems that allow * users to give away files, without introducing a security hole. * Security depends on permissions not containing the setuid or * setgid bits." */ #if !defined(_WIN32) #ifdef ZSTD_HAVE_FCHOWN if (fd >= 0) { res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */ } else #endif { res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */ } #endif res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */ #if !defined(_WIN32) #ifdef ZSTD_HAVE_FCHOWN if (fd >= 0) { res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */ } else #endif { res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */ } #endif errno = 0; UTIL_TRACE_RET(-res); return -res; /* number of errors is returned */ } int UTIL_isDirectory(const char* infilename) { stat_t statbuf; int ret; UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename); ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf); UTIL_TRACE_RET(ret); return ret; } int UTIL_isDirectoryStat(const stat_t* statbuf) { int ret; UTIL_TRACE_CALL("UTIL_isDirectoryStat()"); #if defined(_MSC_VER) ret = (statbuf->st_mode & _S_IFDIR) != 0; #else ret = S_ISDIR(statbuf->st_mode) != 0; #endif UTIL_TRACE_RET(ret); return ret; } int UTIL_compareStr(const void *p1, const void *p2) { return strcmp(* (char * const *) p1, * (char * const *) p2); } int UTIL_isSameFile(const char* fName1, const char* fName2) { int ret; assert(fName1 != NULL); assert(fName2 != NULL); UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2); #if defined(_MSC_VER) || defined(_WIN32) /* note : Visual does not support file identification by inode. * inode does not work on Windows, even with a posix layer, like msys2. * The following work-around is limited to detecting exact name repetition only, * aka `filename` is considered different from `subdir/../filename` */ ret = !strcmp(fName1, fName2); #else { stat_t file1Stat; stat_t file2Stat; ret = UTIL_stat(fName1, &file1Stat) && UTIL_stat(fName2, &file2Stat) && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat); } #endif UTIL_TRACE_RET(ret); return ret; } int UTIL_isSameFileStat( const char* fName1, const char* fName2, const stat_t* file1Stat, const stat_t* file2Stat) { int ret; assert(fName1 != NULL); assert(fName2 != NULL); UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2); #if defined(_MSC_VER) || defined(_WIN32) /* note : Visual does not support file identification by inode. * inode does not work on Windows, even with a posix layer, like msys2. * The following work-around is limited to detecting exact name repetition only, * aka `filename` is considered different from `subdir/../filename` */ (void)file1Stat; (void)file2Stat; ret = !strcmp(fName1, fName2); #else { ret = (file1Stat->st_dev == file2Stat->st_dev) && (file1Stat->st_ino == file2Stat->st_ino); } #endif UTIL_TRACE_RET(ret); return ret; } /* UTIL_isFIFO : distinguish named pipes */ int UTIL_isFIFO(const char* infilename) { UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename); /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ #if PLATFORM_POSIX_VERSION >= 200112L { stat_t statbuf; if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) { UTIL_TRACE_RET(1); return 1; } } #endif (void)infilename; UTIL_TRACE_RET(0); return 0; } /* UTIL_isFIFO : distinguish named pipes */ int UTIL_isFIFOStat(const stat_t* statbuf) { /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ #if PLATFORM_POSIX_VERSION >= 200112L if (S_ISFIFO(statbuf->st_mode)) return 1; #endif (void)statbuf; return 0; } /* UTIL_isBlockDevStat : distinguish named pipes */ int UTIL_isBlockDevStat(const stat_t* statbuf) { /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ #if PLATFORM_POSIX_VERSION >= 200112L if (S_ISBLK(statbuf->st_mode)) return 1; #endif (void)statbuf; return 0; } int UTIL_isLink(const char* infilename) { UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename); /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ #if PLATFORM_POSIX_VERSION >= 200112L { stat_t statbuf; int const r = lstat(infilename, &statbuf); if (!r && S_ISLNK(statbuf.st_mode)) { UTIL_TRACE_RET(1); return 1; } } #endif (void)infilename; UTIL_TRACE_RET(0); return 0; } static int g_fakeStdinIsConsole = 0; static int g_fakeStderrIsConsole = 0; static int g_fakeStdoutIsConsole = 0; int UTIL_isConsole(FILE* file) { int ret; UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file)); if (file == stdin && g_fakeStdinIsConsole) ret = 1; else if (file == stderr && g_fakeStderrIsConsole) ret = 1; else if (file == stdout && g_fakeStdoutIsConsole) ret = 1; else ret = IS_CONSOLE(file); UTIL_TRACE_RET(ret); return ret; } void UTIL_fakeStdinIsConsole(void) { g_fakeStdinIsConsole = 1; } void UTIL_fakeStdoutIsConsole(void) { g_fakeStdoutIsConsole = 1; } void UTIL_fakeStderrIsConsole(void) { g_fakeStderrIsConsole = 1; } U64 UTIL_getFileSize(const char* infilename) { stat_t statbuf; UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename); if (!UTIL_stat(infilename, &statbuf)) { UTIL_TRACE_RET(-1); return UTIL_FILESIZE_UNKNOWN; } { U64 const size = UTIL_getFileSizeStat(&statbuf); UTIL_TRACE_RET((int)size); return size; } } U64 UTIL_getFileSizeStat(const stat_t* statbuf) { if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN; #if defined(_MSC_VER) if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; #elif defined(__MINGW32__) && defined (__MSVCRT__) if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; #else if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN; #endif return (U64)statbuf->st_size; } UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { UTIL_HumanReadableSize_t hrs; if (g_utilDisplayLevel > 3) { /* In verbose mode, do not scale sizes down, except in the case of * values that exceed the integral precision of a double. */ if (size >= (1ull << 53)) { hrs.value = (double)size / (1ull << 20); hrs.suffix = " MiB"; /* At worst, a double representation of a maximal size will be * accurate to better than tens of kilobytes. */ hrs.precision = 2; } else { hrs.value = (double)size; hrs.suffix = " B"; hrs.precision = 0; } } else { /* In regular mode, scale sizes down and use suffixes. */ if (size >= (1ull << 60)) { hrs.value = (double)size / (1ull << 60); hrs.suffix = " EiB"; } else if (size >= (1ull << 50)) { hrs.value = (double)size / (1ull << 50); hrs.suffix = " PiB"; } else if (size >= (1ull << 40)) { hrs.value = (double)size / (1ull << 40); hrs.suffix = " TiB"; } else if (size >= (1ull << 30)) { hrs.value = (double)size / (1ull << 30); hrs.suffix = " GiB"; } else if (size >= (1ull << 20)) { hrs.value = (double)size / (1ull << 20); hrs.suffix = " MiB"; } else if (size >= (1ull << 10)) { hrs.value = (double)size / (1ull << 10); hrs.suffix = " KiB"; } else { hrs.value = (double)size; hrs.suffix = " B"; } if (hrs.value >= 100 || (U64)hrs.value == size) { hrs.precision = 0; } else if (hrs.value >= 10) { hrs.precision = 1; } else if (hrs.value > 1) { hrs.precision = 2; } else { hrs.precision = 3; } } return hrs; } U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) { U64 total = 0; unsigned n; UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles); for (n=0; n= 1) perror("zstd:util:readLinesFromFile"); return -1; } while ( !feof(inputFile) ) { size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile); if (lineLength == 0) break; assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */ pos += lineLength; ++nbFiles; } CONTROL( fclose(inputFile) == 0 ); return nbFiles; } /*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/ FileNamesTable* UTIL_createFileNamesTable_fromFileName(const char* inputFileName) { size_t nbFiles = 0; char* buf; size_t bufSize; size_t pos = 0; stat_t statbuf; if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf)) return NULL; { U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf); if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE) return NULL; bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */ } buf = (char*) malloc(bufSize); CONTROL( buf != NULL ); { int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName); if (ret_nbFiles <= 0) { free(buf); return NULL; } nbFiles = (size_t)ret_nbFiles; } { const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable)); CONTROL(filenamesTable != NULL); { size_t fnb; for (fnb = 0, pos = 0; fnb < nbFiles; fnb++) { filenamesTable[fnb] = buf+pos; pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */ } } assert(pos <= bufSize); return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf); } } static FileNamesTable* UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf) { FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table)); CONTROL(table != NULL); table->fileNames = filenames; table->buf = buf; table->tableSize = tableSize; table->tableCapacity = tableCapacity; return table; } FileNamesTable* UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf) { return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf); } void UTIL_freeFileNamesTable(FileNamesTable* table) { if (table==NULL) return; free((void*)table->fileNames); free(table->buf); free(table); } FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize) { const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable)); FileNamesTable* fnt; if (fnTable==NULL) return NULL; fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL); fnt->tableSize = 0; /* the table is empty */ return fnt; } int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) { size_t i; for(i=0 ;i < table->tableSize; i++) { if(!strcmp(table->fileNames[i], name)) { return (int)i; } } return -1; } void UTIL_refFilename(FileNamesTable* fnt, const char* filename) { assert(fnt->tableSize < fnt->tableCapacity); fnt->fileNames[fnt->tableSize] = filename; fnt->tableSize++; } static size_t getTotalTableSize(FileNamesTable* table) { size_t fnb = 0, totalSize = 0; for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) { totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */ } return totalSize; } FileNamesTable* UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2) { unsigned newTableIdx = 0; size_t pos = 0; size_t newTotalTableSize; char* buf; FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL); CONTROL( newTable != NULL ); newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2); buf = (char*) calloc(newTotalTableSize, sizeof(*buf)); CONTROL ( buf != NULL ); newTable->buf = buf; newTable->tableSize = table1->tableSize + table2->tableSize; newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames))); CONTROL ( newTable->fileNames != NULL ); { unsigned idx1; for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) { size_t const curLen = strlen(table1->fileNames[idx1]); memcpy(buf+pos, table1->fileNames[idx1], curLen); assert(newTableIdx <= newTable->tableSize); newTable->fileNames[newTableIdx] = buf+pos; pos += curLen+1; } } { unsigned idx2; for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) { size_t const curLen = strlen(table2->fileNames[idx2]); memcpy(buf+pos, table2->fileNames[idx2], curLen); assert(newTableIdx < newTable->tableSize); newTable->fileNames[newTableIdx] = buf+pos; pos += curLen+1; } } assert(pos <= newTotalTableSize); newTable->tableSize = newTableIdx; UTIL_freeFileNamesTable(table1); UTIL_freeFileNamesTable(table2); return newTable; } #ifdef _WIN32 static int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { char* path; size_t dirLength, pathLength; int nbFiles = 0; WIN32_FIND_DATAA cFile; HANDLE hFile; dirLength = strlen(dirName); path = (char*) malloc(dirLength + 3); if (!path) return 0; memcpy(path, dirName, dirLength); path[dirLength] = '\\'; path[dirLength+1] = '*'; path[dirLength+2] = 0; hFile=FindFirstFileA(path, &cFile); if (hFile == INVALID_HANDLE_VALUE) { UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName); return 0; } free(path); do { size_t const fnameLength = strlen(cFile.cFileName); path = (char*) malloc(dirLength + fnameLength + 2); if (!path) { FindClose(hFile); return 0; } memcpy(path, dirName, dirLength); path[dirLength] = '\\'; memcpy(path+dirLength+1, cFile.cFileName, fnameLength); pathLength = dirLength+1+fnameLength; path[pathLength] = 0; if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { if ( strcmp (cFile.cFileName, "..") == 0 || strcmp (cFile.cFileName, ".") == 0 ) continue; /* Recursively call "UTIL_prepareFileList" with the new path. */ nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) { if (*bufStart + *pos + pathLength >= *bufEnd) { ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } *bufEnd = *bufStart + newListSize; } if (*bufStart + *pos + pathLength < *bufEnd) { memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */); *pos += pathLength + 1; nbFiles++; } } free(path); } while (FindNextFileA(hFile, &cFile)); FindClose(hFile); return nbFiles; } #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ static int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { DIR* dir; struct dirent * entry; size_t dirLength; int nbFiles = 0; if (!(dir = opendir(dirName))) { UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); return 0; } dirLength = strlen(dirName); errno = 0; while ((entry = readdir(dir)) != NULL) { char* path; size_t fnameLength, pathLength; if (strcmp (entry->d_name, "..") == 0 || strcmp (entry->d_name, ".") == 0) continue; fnameLength = strlen(entry->d_name); path = (char*) malloc(dirLength + fnameLength + 2); if (!path) { closedir(dir); return 0; } memcpy(path, dirName, dirLength); path[dirLength] = '/'; memcpy(path+dirLength+1, entry->d_name, fnameLength); pathLength = dirLength+1+fnameLength; path[pathLength] = 0; if (!followLinks && UTIL_isLink(path)) { UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); free(path); continue; } if (UTIL_isDirectory(path)) { nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ if (*bufStart == NULL) { free(path); closedir(dir); return 0; } } else { if (*bufStart + *pos + pathLength >= *bufEnd) { ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; assert(newListSize >= 0); *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); if (*bufStart != NULL) { *bufEnd = *bufStart + newListSize; } else { free(path); closedir(dir); return 0; } } if (*bufStart + *pos + pathLength < *bufEnd) { memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ *pos += pathLength + 1; nbFiles++; } } free(path); errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ } if (errno != 0) { UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno)); free(*bufStart); *bufStart = NULL; } closedir(dir); return nbFiles; } #else static int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks; UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName); return 0; } #endif /* #ifdef _WIN32 */ int UTIL_isCompressedFile(const char *inputName, const char *extensionList[]) { const char* ext = UTIL_getFileExtension(inputName); while(*extensionList!=NULL) { const int isCompressedExtension = strcmp(ext,*extensionList); if(isCompressedExtension==0) return 1; ++extensionList; } return 0; } /*Utility function to get file extension from file */ const char* UTIL_getFileExtension(const char* infilename) { const char* extension = strrchr(infilename, '.'); if(!extension || extension==infilename) return ""; return extension; } static int pathnameHas2Dots(const char *pathname) { /* We need to figure out whether any ".." present in the path is a whole * path token, which is the case if it is bordered on both sides by either * the beginning/end of the path or by a directory separator. */ const char *needle = pathname; while (1) { needle = strstr(needle, ".."); if (needle == NULL) { return 0; } if ((needle == pathname || needle[-1] == PATH_SEP) && (needle[2] == '\0' || needle[2] == PATH_SEP)) { return 1; } /* increment so we search for the next match */ needle++; }; return 0; } static int isFileNameValidForMirroredOutput(const char *filename) { return !pathnameHas2Dots(filename); } #define DIR_DEFAULT_MODE 0755 static mode_t getDirMode(const char *dirName) { stat_t st; if (!UTIL_stat(dirName, &st)) { UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno)); return DIR_DEFAULT_MODE; } if (!UTIL_isDirectoryStat(&st)) { UTIL_DISPLAY("zstd: expected directory: %s\n", dirName); return DIR_DEFAULT_MODE; } return st.st_mode; } static int makeDir(const char *dir, mode_t mode) { #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) int ret = _mkdir(dir); (void) mode; #else int ret = mkdir(dir, mode); #endif if (ret != 0) { if (errno == EEXIST) return 0; UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno)); } return ret; } /* this function requires a mutable input string */ static void convertPathnameToDirName(char *pathname) { size_t len = 0; char* pos = NULL; /* get dir name from pathname similar to 'dirname()' */ assert(pathname != NULL); /* remove trailing '/' chars */ len = strlen(pathname); assert(len > 0); while (pathname[len] == PATH_SEP) { pathname[len] = '\0'; len--; } if (len == 0) return; /* if input is a single file, return '.' instead. i.e. * "xyz/abc/file.txt" => "xyz/abc" "./file.txt" => "." "file.txt" => "." */ pos = strrchr(pathname, PATH_SEP); if (pos == NULL) { pathname[0] = '.'; pathname[1] = '\0'; } else { *pos = '\0'; } } /* pathname must be valid */ static const char* trimLeadingRootChar(const char *pathname) { assert(pathname != NULL); if (pathname[0] == PATH_SEP) return pathname + 1; return pathname; } /* pathname must be valid */ static const char* trimLeadingCurrentDirConst(const char *pathname) { assert(pathname != NULL); if ((pathname[0] == '.') && (pathname[1] == PATH_SEP)) return pathname + 2; return pathname; } static char* trimLeadingCurrentDir(char *pathname) { /* 'union charunion' can do const-cast without compiler warning */ union charunion { char *chr; const char* cchr; } ptr; ptr.cchr = trimLeadingCurrentDirConst(pathname); return ptr.chr; } /* remove leading './' or '/' chars here */ static const char * trimPath(const char *pathname) { return trimLeadingRootChar( trimLeadingCurrentDirConst(pathname)); } static char* mallocAndJoin2Dir(const char *dir1, const char *dir2) { assert(dir1 != NULL && dir2 != NULL); { const size_t dir1Size = strlen(dir1); const size_t dir2Size = strlen(dir2); char *outDirBuffer, *buffer; outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2); CONTROL(outDirBuffer != NULL); memcpy(outDirBuffer, dir1, dir1Size); outDirBuffer[dir1Size] = '\0'; if (dir2[0] == '.') return outDirBuffer; buffer = outDirBuffer + dir1Size; if (dir1Size > 0 && *(buffer - 1) != PATH_SEP) { *buffer = PATH_SEP; buffer++; } memcpy(buffer, dir2, dir2Size); buffer[dir2Size] = '\0'; return outDirBuffer; } } /* this function will return NULL if input srcFileName is not valid name for mirrored output path */ char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName) { char* pathname = NULL; if (!isFileNameValidForMirroredOutput(srcFileName)) return NULL; pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName)); convertPathnameToDirName(pathname); return pathname; } static int mirrorSrcDir(char* srcDirName, const char* outDirName) { mode_t srcMode; int status = 0; char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName)); if (!newDir) return -ENOMEM; srcMode = getDirMode(srcDirName); status = makeDir(newDir, srcMode); free(newDir); return status; } static int mirrorSrcDirRecursive(char* srcDirName, const char* outDirName) { int status = 0; char* pp = trimLeadingCurrentDir(srcDirName); char* sp = NULL; while ((sp = strchr(pp, PATH_SEP)) != NULL) { if (sp != pp) { *sp = '\0'; status = mirrorSrcDir(srcDirName, outDirName); if (status != 0) return status; *sp = PATH_SEP; } pp = sp + 1; } status = mirrorSrcDir(srcDirName, outDirName); return status; } static void makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName) { unsigned int i = 0; for (i = 0; i < nbFile; i++) mirrorSrcDirRecursive(srcDirNames[i], outDirName); } static int firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir) { size_t firstDirLen = strlen(firstDir), secondDirLen = strlen(secondDir); return firstDirLen <= secondDirLen && (secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') && 0 == strncmp(firstDir, secondDir, firstDirLen); } static int compareDir(const void* pathname1, const void* pathname2) { /* sort it after remove the leading '/' or './'*/ const char* s1 = trimPath(*(char * const *) pathname1); const char* s2 = trimPath(*(char * const *) pathname2); return strcmp(s1, s2); } static void makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName) { unsigned int i = 0, uniqueDirNr = 0; char** uniqueDirNames = NULL; if (nbFile == 0) return; uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *)); CONTROL(uniqueDirNames != NULL); /* if dirs is "a/b/c" and "a/b/c/d", we only need call: * we just need "a/b/c/d" */ qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir); uniqueDirNr = 1; uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0]; for (i = 1; i < nbFile; i++) { char* prevDirName = srcDirNames[i - 1]; char* currDirName = srcDirNames[i]; /* note: we always compare trimmed path, i.e.: * src dir of "./foo" and "/foo" will be both saved into: * "outDirName/foo/" */ if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName), trimPath(currDirName))) uniqueDirNr++; /* we need to maintain original src dir name instead of trimmed * dir, so we can retrieve the original src dir's mode_t */ uniqueDirNames[uniqueDirNr - 1] = currDirName; } makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName); free(uniqueDirNames); } static void makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName) { unsigned int i = 0; for (i = 0; i < nbFile; ++i) convertPathnameToDirName(srcFileNames[i]); makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName); } void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName) { unsigned int i = 0, validFilenamesNr = 0; char** srcFileNames = (char **) malloc(nbFile * sizeof (char *)); CONTROL(srcFileNames != NULL); /* check input filenames is valid */ for (i = 0; i < nbFile; ++i) { if (isFileNameValidForMirroredOutput(inFileNames[i])) { char* fname = STRDUP(inFileNames[i]); CONTROL(fname != NULL); srcFileNames[validFilenamesNr++] = fname; } } if (validFilenamesNr > 0) { makeDir(outDirName, DIR_DEFAULT_MODE); makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName); } for (i = 0; i < validFilenamesNr; i++) free(srcFileNames[i]); free(srcFileNames); } FileNamesTable* UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks) { unsigned nbFiles; char* buf = (char*)malloc(LIST_SIZE_INCREASE); char* bufend = buf + LIST_SIZE_INCREASE; if (!buf) return NULL; { size_t ifnNb, pos; for (ifnNb=0, pos=0, nbFiles=0; ifnNb= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; assert(newListSize >= 0); buf = (char*)UTIL_realloc(buf, (size_t)newListSize); if (!buf) return NULL; bufend = buf + newListSize; } if (buf + pos + len < bufend) { memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */ pos += len + 1; nbFiles++; } } else { nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks); if (buf == NULL) return NULL; } } } /* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */ { size_t ifnNb, pos; size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */ const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable)); if (!fileNamesTable) { free(buf); return NULL; } for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) { fileNamesTable[ifnNb] = buf + pos; if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; } pos += strlen(fileNamesTable[ifnNb]) + 1; } return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf); } } void UTIL_expandFNT(FileNamesTable** fnt, int followLinks) { FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks); CONTROL(newFNT != NULL); UTIL_freeFileNamesTable(*fnt); *fnt = newFNT; } FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames) { size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames); const char** const newFNTable = (const char**)malloc(sizeof_FNTable); if (newFNTable==NULL) return NULL; memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */ return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL); } /*-**************************************** * count the number of cores ******************************************/ #if defined(_WIN32) || defined(WIN32) #include typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); DWORD CountSetBits(ULONG_PTR bitMask) { DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; DWORD bitSetCount = 0; ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; DWORD i; for (i = 0; i <= LSHIFT; ++i) { bitSetCount += ((bitMask & bitTest)?1:0); bitTest/=2; } return bitSetCount; } int UTIL_countCores(int logical) { static int numCores = 0; if (numCores != 0) return numCores; { LPFN_GLPI glpi; BOOL done = FALSE; PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; DWORD returnLength = 0; size_t byteOffset = 0; #if defined(_MSC_VER) /* Visual Studio does not like the following cast */ # pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */ # pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */ #endif glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")), "GetLogicalProcessorInformation"); if (glpi == NULL) { goto failed; } while(!done) { DWORD rc = glpi(buffer, &returnLength); if (FALSE == rc) { if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { if (buffer) free(buffer); buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); if (buffer == NULL) { perror("zstd"); exit(1); } } else { /* some other error */ goto failed; } } else { done = TRUE; } } ptr = buffer; while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { if (ptr->Relationship == RelationProcessorCore) { if (logical) numCores += CountSetBits(ptr->ProcessorMask); else numCores++; } ptr++; byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); } free(buffer); return numCores; } failed: /* try to fall back on GetSystemInfo */ { SYSTEM_INFO sysinfo; GetSystemInfo(&sysinfo); numCores = sysinfo.dwNumberOfProcessors; if (numCores == 0) numCores = 1; /* just in case */ } return numCores; } #elif defined(__APPLE__) #include /* Use apple-provided syscall * see: man 3 sysctl */ int UTIL_countCores(int logical) { static S32 numCores = 0; /* apple specifies int32_t */ if (numCores != 0) return numCores; { size_t size = sizeof(S32); int const ret = sysctlbyname(logical ? "hw.logicalcpu" : "hw.physicalcpu", &numCores, &size, NULL, 0); if (ret != 0) { if (errno == ENOENT) { /* entry not present, fall back on 1 */ numCores = 1; } else { perror("zstd: can't get number of cpus"); exit(1); } } return numCores; } } #elif defined(__linux__) /* parse /proc/cpuinfo * siblings / cpu cores should give hyperthreading ratio * otherwise fall back on sysconf */ int UTIL_countCores(int logical) { static int numCores = 0; if (numCores != 0) return numCores; numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); if (numCores == -1) { /* value not queryable, fall back on 1 */ return numCores = 1; } /* try to determine if there's hyperthreading */ { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); #define BUF_SIZE 80 char buff[BUF_SIZE]; int siblings = 0; int cpu_cores = 0; int ratio = 1; if (cpuinfo == NULL) { /* fall back on the sysconf value */ return numCores; } /* assume the cpu cores/siblings values will be constant across all * present processors */ while (!feof(cpuinfo)) { if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { if (strncmp(buff, "siblings", 8) == 0) { const char* const sep = strchr(buff, ':'); if (sep == NULL || *sep == '\0') { /* formatting was broken? */ goto failed; } siblings = atoi(sep + 1); } if (strncmp(buff, "cpu cores", 9) == 0) { const char* const sep = strchr(buff, ':'); if (sep == NULL || *sep == '\0') { /* formatting was broken? */ goto failed; } cpu_cores = atoi(sep + 1); } } else if (ferror(cpuinfo)) { /* fall back on the sysconf value */ goto failed; } } if (siblings && cpu_cores && siblings > cpu_cores) { ratio = siblings / cpu_cores; } if (ratio && numCores > ratio && !logical) { numCores = numCores / ratio; } failed: fclose(cpuinfo); return numCores; } } #elif defined(__FreeBSD__) #include #include /* Use physical core sysctl when available * see: man 4 smp, man 3 sysctl */ int UTIL_countCores(int logical) { static int numCores = 0; /* freebsd sysctl is native int sized */ #if __FreeBSD_version >= 1300008 static int perCore = 1; #endif if (numCores != 0) return numCores; #if __FreeBSD_version >= 1300008 { size_t size = sizeof(numCores); int ret = sysctlbyname("kern.smp.cores", &numCores, &size, NULL, 0); if (ret == 0) { if (logical) { ret = sysctlbyname("kern.smp.threads_per_core", &perCore, &size, NULL, 0); /* default to physical cores if logical cannot be read */ if (ret == 0) numCores *= perCore; } return numCores; } if (errno != ENOENT) { perror("zstd: can't get number of cpus"); exit(1); } /* sysctl not present, fall through to older sysconf method */ } #else /* suppress unused parameter warning */ (void) logical; #endif numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); if (numCores == -1) { /* value not queryable, fall back on 1 */ numCores = 1; } return numCores; } #elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__) /* Use POSIX sysconf * see: man 3 sysconf */ int UTIL_countCores(int logical) { static int numCores = 0; /* suppress unused parameter warning */ (void)logical; if (numCores != 0) return numCores; numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); if (numCores == -1) { /* value not queryable, fall back on 1 */ return numCores = 1; } return numCores; } #else int UTIL_countCores(int logical) { /* suppress unused parameter warning */ (void)logical; /* assume 1 */ return 1; } #endif int UTIL_countPhysicalCores(void) { return UTIL_countCores(0); } int UTIL_countLogicalCores(void) { return UTIL_countCores(1); } #if defined (__cplusplus) } #endif