From: Charles Manning Date: Wed, 21 Dec 2011 02:23:19 +0000 (+1300) Subject: large file storage WIP. Can save and verify a large file under yaffs direct. X-Git-Tag: pre-driver-refactoring~34^2~4 X-Git-Url: http://www.aleph1.co.uk/gitweb/?p=yaffs2.git;a=commitdiff_plain;h=fc4028b21ae665f45c149e66b10d13374721383c large file storage WIP. Can save and verify a large file under yaffs direct. Signed-off-by: Charles Manning --- diff --git a/direct/basic-test/dtest.c b/direct/basic-test/dtest.c index fa48158..edf9d63 100644 --- a/direct/basic-test/dtest.c +++ b/direct/basic-test/dtest.c @@ -533,7 +533,7 @@ void dumpDirFollow(const char *dname) yaffs_lstat(str,&s); - printf("%s ino %d length %d mode %X ",de->d_name,(int)s.st_ino,(int)s.st_size,s.st_mode); + printf("%s ino %lld length %d mode %X ",de->d_name,(int)s.st_ino,s.st_size,s.st_mode); switch(s.st_mode & S_IFMT) { case S_IFREG: printf("data file"); break; @@ -580,7 +580,8 @@ void dump_directory_tree_worker(const char *dname,int recursive) yaffs_lstat(str,&s); - printf("%s inode %d obj %x length %d mode %X ",str,s.st_ino,de->d_dont_use,(int)s.st_size,s.st_mode); + printf("%s inode %d obj %x length %lld mode %X ", + str,s.st_ino,de->d_dont_use, s.st_size,s.st_mode); switch(s.st_mode & S_IFMT) { case S_IFREG: printf("data file"); break; @@ -2755,6 +2756,151 @@ void link_follow_test(const char *mountpt) } + +#define N_WRITES 2000 +#define STRIDE 2000 + +#define BUFFER_N 1100 +unsigned xxbuffer[BUFFER_N]; + + +void set_buffer(int n) +{ + int i; + for(i = 0; i < BUFFER_N; i++) + xxbuffer[i] = i + n; +} + +void write_big_sparse_file(int h) +{ + int i; + loff_t offset = 0; + loff_t pos; + int n = sizeof(xxbuffer); + int wrote; + + for(i = 0; i < N_WRITES; i++) { + printf("writing at %lld\n", offset); + set_buffer(i); + pos = yaffs_lseek(h, offset, SEEK_SET); + if(pos != offset) { + printf("mismatched seek pos %lld offset %lld\n", + pos, offset); + perror("lseek64"); + exit(1); + } + wrote = yaffs_write(h, xxbuffer, n); + + if(wrote != n) { + printf("mismatched write wrote %d n %d\n", wrote, n); + exit(1); + } + + offset += (STRIDE * sizeof(xxbuffer)); + } + + yaffs_ftruncate(h, offset); + +} + + + + +void verify_big_sparse_file(int h) +{ + unsigned check_buffer[BUFFER_N]; + int i; + loff_t offset = 0; + loff_t pos; + int n = sizeof(check_buffer); + int result; + const char * check_type; + int checks_failed = 0; + int checks_passed = 0; + + for(i = 0; i < N_WRITES * STRIDE; i++) { + if(i % STRIDE) { + check_type = "zero"; + memset(xxbuffer,0, n); + } else { + check_type = "buffer"; + set_buffer(i/STRIDE); + } + printf("%s checking %lld\n", check_type, offset); + pos = yaffs_lseek(h, offset, SEEK_SET); + if(pos != offset) { + printf("mismatched seek pos %lld offset %lld\n", + pos, offset); + perror("lseek64"); + exit(1); + } + result = yaffs_read(h, check_buffer, n); + + if(result != n) { + printf("mismatched read result %d n %d\n", result, n); + exit(1); + } + + + + + if(memcmp(xxbuffer, check_buffer, n)) { + int j; + + printf("buffer at %lld mismatches\n", pos); + printf("xxbuffer "); + for(j = 0; j < 20; j++) + printf(" %d",xxbuffer[j]); + printf("\n"); + printf("check_buffer "); + for(j = 0; j < 20; j++) + printf(" %d",check_buffer[j]); + printf("\n"); + + checks_failed++; + } else { + checks_passed++; + } + + offset += sizeof(xxbuffer); + } + + printf("%d checks passed, %d checks failed\n", checks_passed, checks_failed); + +} + + +void large_file_test(const char *mountpt) +{ + int handle; + char fullname[100]; + + yaffs_trace_mask = 0; + + yaffs_start_up(); + + yaffs_mount(mountpt); + printf("mounted\n"); + dumpDir(mountpt); + + sprintf(fullname, "%s/%s", mountpt, "big-test-file"); + handle = yaffs_open(fullname, O_CREAT | O_RDWR | O_TRUNC, S_IREAD | S_IWRITE); + + if(handle < 0) { + perror("opening file"); + exit(1); + } + + write_big_sparse_file(handle); + verify_big_sparse_file(handle); + + printf("Job done\n"); + yaffs_unmount(mountpt); + + +} + + int random_seed; int simulate_power_failure; @@ -2820,7 +2966,10 @@ int main(int argc, char *argv[]) //test_flash_traffic("yaffs2"); // link_follow_test("/yaffs2"); - basic_utime_test("/yaffs2"); + + large_file_test("/yaffs2"); + + //basic_utime_test("/yaffs2"); return 0; diff --git a/direct/yaffsfs.c b/direct/yaffsfs.c index 568fe1e..290bcf7 100644 --- a/direct/yaffsfs.c +++ b/direct/yaffsfs.c @@ -75,7 +75,7 @@ typedef struct{ u8 shareWrite:1; int inodeId:12; /* Index to corresponding yaffsfs_Inode */ int handleCount:10; /* Number of handles for this fd */ - u32 position; /* current position in file */ + loff_t position; /* current position in file */ }yaffsfs_FileDes; typedef struct { @@ -1037,17 +1037,17 @@ int yaffs_close(int handle) -int yaffsfs_do_read(int handle, void *vbuf, unsigned int nbyte, int isPread, int offset) +int yaffsfs_do_read(int handle, void *vbuf, unsigned int nbyte, int isPread, loff_t offset) { yaffsfs_FileDes *fd = NULL; struct yaffs_obj *obj = NULL; - int pos = 0; - int startPos = 0; - int endPos = 0; + loff_t pos = 0; + loff_t startPos = 0; + loff_t endPos = 0; int nRead = 0; int nToRead = 0; int totalRead = 0; - unsigned int maxRead; + loff_t maxRead; u8 *buf = (u8 *)vbuf; if(!vbuf){ @@ -1154,18 +1154,18 @@ int yaffs_read(int handle, void *buf, unsigned int nbyte) return yaffsfs_do_read(handle, buf, nbyte, 0, 0); } -int yaffs_pread(int handle, void *buf, unsigned int nbyte, unsigned int offset) +int yaffs_pread(int handle, void *buf, unsigned int nbyte, loff_t offset) { return yaffsfs_do_read(handle, buf, nbyte, 1, offset); } -int yaffsfs_do_write(int handle, const void *vbuf, unsigned int nbyte, int isPwrite, int offset) +int yaffsfs_do_write(int handle, const void *vbuf, unsigned int nbyte, int isPwrite, loff_t offset) { yaffsfs_FileDes *fd = NULL; struct yaffs_obj *obj = NULL; - int pos = 0; - int startPos = 0; - int endPos; + loff_t pos = 0; + loff_t startPos = 0; + loff_t endPos; int nWritten = 0; int totalWritten = 0; int write_trhrough = 0; @@ -1268,13 +1268,13 @@ int yaffs_write(int fd, const void *buf, unsigned int nbyte) return yaffsfs_do_write(fd, buf, nbyte, 0, 0); } -int yaffs_pwrite(int fd, const void *buf, unsigned int nbyte, unsigned int offset) +int yaffs_pwrite(int fd, const void *buf, unsigned int nbyte, loff_t offset) { return yaffsfs_do_write(fd, buf, nbyte, 1, offset); } -int yaffs_truncate(const YCHAR *path,off_t new_size) +int yaffs_truncate(const YCHAR *path,loff_t new_size) { struct yaffs_obj *obj = NULL; struct yaffs_obj *dir = NULL; @@ -1312,14 +1312,14 @@ int yaffs_truncate(const YCHAR *path,off_t new_size) else if(new_size < 0 || new_size > YAFFS_MAX_FILE_SIZE) yaffsfs_SetError(-EINVAL); else - result = yaffs_resize_file(obj,new_size); + result = yaffs_resize_file(obj, new_size); yaffsfs_Unlock(); return (result) ? 0 : -1; } -int yaffs_ftruncate(int handle, off_t new_size) +int yaffs_ftruncate(int handle, loff_t new_size) { yaffsfs_FileDes *fd = NULL; struct yaffs_obj *obj = NULL; @@ -1340,19 +1340,19 @@ int yaffs_ftruncate(int handle, off_t new_size) yaffsfs_SetError(-EINVAL); else /* resize the file */ - result = yaffs_resize_file(obj,new_size); + result = yaffs_resize_file(obj, new_size); yaffsfs_Unlock(); return (result) ? 0 : -1; } -off_t yaffs_lseek(int handle, off_t offset, int whence) +loff_t yaffs_lseek(int handle, loff_t offset, int whence) { yaffsfs_FileDes *fd = NULL; struct yaffs_obj *obj = NULL; - int pos = -1; - int fSize = -1; + loff_t pos = -1; + loff_t fSize = -1; yaffsfs_Lock(); fd = yaffsfs_HandleToFileDes(handle); diff --git a/direct/yaffsfs.h b/direct/yaffsfs.h index 6b6b328..4310316 100644 --- a/direct/yaffsfs.h +++ b/direct/yaffsfs.h @@ -35,13 +35,13 @@ #define NAME_MAX 256 #endif -#define YAFFS_MAX_FILE_SIZE (0x7FFFFFFF) +#define YAFFS_MAX_FILE_SIZE (0x800000000LL - 1) struct yaffs_dirent{ long d_ino; /* inode number */ off_t d_off; /* offset to this dirent */ - unsigned short d_reclen; /* length of this d_name */ + unsigned short d_reclen; /* length of this dirent */ YUCHAR d_type; /* type of this record */ YCHAR d_name [NAME_MAX+1]; /* file name (null-terminated) */ unsigned d_dont_use; /* debug pointer, not for public consumption */ @@ -55,14 +55,14 @@ typedef struct __opaque yaffs_DIR; struct yaffs_stat{ - int st_dev; /* device */ + int st_dev; /* device */ int st_ino; /* inode */ unsigned st_mode; /* protection */ int st_nlink; /* number of hard links */ int st_uid; /* user ID of owner */ int st_gid; /* group ID of owner */ unsigned st_rdev; /* device type (if inode device) */ - off_t st_size; /* total size, in bytes */ + loff_t st_size; /* total size, in bytes */ unsigned long st_blksize; /* blocksize for filesystem I/O */ unsigned long st_blocks; /* number of blocks allocated */ #ifdef CONFIG_YAFFS_WINCE @@ -71,7 +71,7 @@ struct yaffs_stat{ unsigned long yst_wince_mtime[2]; unsigned long yst_wince_ctime[2]; #else - unsigned long yst_atime; /* time of last access */ + unsigned long yst_atime; /* time of last access */ unsigned long yst_mtime; /* time of last modification */ unsigned long yst_ctime; /* time of last change */ #endif @@ -98,13 +98,13 @@ int yaffs_dup(int fd); int yaffs_read(int fd, void *buf, unsigned int nbyte) ; int yaffs_write(int fd, const void *buf, unsigned int nbyte) ; -int yaffs_pread(int fd, void *buf, unsigned int nbyte, unsigned int offset); -int yaffs_pwrite(int fd, const void *buf, unsigned int nbyte, unsigned int offset); +int yaffs_pread(int fd, void *buf, unsigned int nbyte, loff_t offset); +int yaffs_pwrite(int fd, const void *buf, unsigned int nbyte, loff_t offset); -off_t yaffs_lseek(int fd, off_t offset, int whence) ; +loff_t yaffs_lseek(int fd, loff_t offset, int whence) ; -int yaffs_truncate(const YCHAR *path, off_t new_size); -int yaffs_ftruncate(int fd, off_t new_size); +int yaffs_truncate(const YCHAR *path, loff_t new_size); +int yaffs_ftruncate(int fd, loff_t new_size); int yaffs_unlink(const YCHAR *path) ; int yaffs_rename(const YCHAR *oldPath, const YCHAR *newPath) ; diff --git a/linux-tests/writebigsparse.c b/linux-tests/writebigsparse.c index 0d70128..fab2d16 100644 --- a/linux-tests/writebigsparse.c +++ b/linux-tests/writebigsparse.c @@ -36,6 +36,7 @@ void write_big_sparse_file(int h) if(pos != offset) { printf("mismatched seek pos %lld offset %lld\n", pos, offset); + perror("lseek64"); exit(1); } wrote = write(h, buffer, n); diff --git a/yaffs_guts.c b/yaffs_guts.c index 79a7988..6166d98 100644 --- a/yaffs_guts.c +++ b/yaffs_guts.c @@ -3504,7 +3504,7 @@ int yaffs_file_rd(struct yaffs_obj *in, u8 * buffer, loff_t offset, int n_bytes) } int yaffs_do_file_wr(struct yaffs_obj *in, const u8 *buffer, loff_t offset, - int n_bytes, int write_trhrough) + int n_bytes, int write_through) { int chunk; @@ -3513,10 +3513,10 @@ int yaffs_do_file_wr(struct yaffs_obj *in, const u8 *buffer, loff_t offset, int n = n_bytes; int n_done = 0; int n_writeback; - int start_write = offset; + loff_t start_write = offset; int chunk_written = 0; u32 n_bytes_read; - u32 chunk_start; + loff_t chunk_start; struct yaffs_dev *dev; dev = in->my_dev; @@ -3524,11 +3524,11 @@ int yaffs_do_file_wr(struct yaffs_obj *in, const u8 *buffer, loff_t offset, while (n > 0 && chunk_written >= 0) { yaffs_addr_to_chunk(dev, offset, &chunk, &start); - if (chunk * dev->data_bytes_per_chunk + start != offset || + if (((loff_t)chunk) * dev->data_bytes_per_chunk + start != offset || start >= dev->data_bytes_per_chunk) { yaffs_trace(YAFFS_TRACE_ERROR, - "AddrToChunk of offset %d gives chunk %d start %d", - (int)offset, chunk, start); + "AddrToChunk of offset %lld gives chunk %d start %d", + offset, chunk, start); } chunk++; /* File pos to chunk in file offset */ @@ -3545,7 +3545,7 @@ int yaffs_do_file_wr(struct yaffs_obj *in, const u8 *buffer, loff_t offset, * before. */ - chunk_start = ((chunk - 1) * dev->data_bytes_per_chunk); + chunk_start = (((loff_t)(chunk - 1)) * dev->data_bytes_per_chunk); if (chunk_start > in->variant.file_variant.file_size) n_bytes_read = 0; /* Past end of file */ @@ -3613,7 +3613,7 @@ int yaffs_do_file_wr(struct yaffs_obj *in, const u8 *buffer, loff_t offset, cache->locked = 0; cache->n_bytes = n_writeback; - if (write_trhrough) { + if (write_through) { chunk_written = yaffs_wr_data_obj (cache->object, @@ -3673,10 +3673,10 @@ int yaffs_do_file_wr(struct yaffs_obj *in, const u8 *buffer, loff_t offset, } int yaffs_wr_file(struct yaffs_obj *in, const u8 *buffer, loff_t offset, - int n_bytes, int write_trhrough) + int n_bytes, int write_through) { yaffs2_handle_hole(in, offset); - return yaffs_do_file_wr(in, buffer, offset, n_bytes, write_trhrough); + return yaffs_do_file_wr(in, buffer, offset, n_bytes, write_through); } /* ---------------------- File resizing stuff ------------------ */ @@ -5002,7 +5002,7 @@ void yaffs_oh_size_load(struct yaffs_obj_hdr *oh, loff_t fsize) loff_t yaffs_oh_to_size(struct yaffs_obj_hdr *oh) { loff_t retval; - + if(~(oh->file_size_high)) retval = (((loff_t) oh->file_size_high) << 32) | (((loff_t) oh->file_size_low) & 0xFFFFFFFF); diff --git a/yaffs_guts.h b/yaffs_guts.h index a063776..72e86a8 100644 --- a/yaffs_guts.h +++ b/yaffs_guts.h @@ -49,7 +49,10 @@ #define YAFFS_TNODES_INTERNAL_BITS (YAFFS_TNODES_LEVEL0_BITS - 1) #define YAFFS_TNODES_INTERNAL_MASK 0x7 #define YAFFS_TNODES_MAX_LEVEL 8 - +#define YAFFS_TNODES_MAX_BITS (YAFFS_TNODES_LEVEL0_BITS + \ + YAFFS_TNODES_INTERNAL_BITS * \ + YAFFS_TNODES_MAX_LEVEL) +#define YAFFS_MAX_CHUNK_ID ((1 << YAFFS_TNODES_MAX_BITS) - 1) /* Constants for YAFFS1 mode */ #define YAFFS_BYTES_PER_SPARE 16 @@ -61,7 +64,7 @@ #define YAFFS_MIN_YAFFS2_CHUNK_SIZE 1024 #define YAFFS_MIN_YAFFS2_SPARE_SIZE 32 -#define YAFFS_MAX_CHUNK_ID 0x000fffff + #define YAFFS_ALLOCATION_NOBJECTS 100 #define YAFFS_ALLOCATION_NTNODES 100 diff --git a/yaffs_vfs_multi.c b/yaffs_vfs_multi.c index 8773b76..ec06418 100644 --- a/yaffs_vfs_multi.c +++ b/yaffs_vfs_multi.c @@ -2815,6 +2815,8 @@ static struct super_block *yaffs_internal_read_super(int yaffs_version, sb->u.generic_sbp = dev; #endif + sb->s_maxbytes = 32000000000UL; + dev->driver_context = mtd; param->name = mtd->name; diff --git a/yaffs_yaffs2.c b/yaffs_yaffs2.c index d580f72..aa32055 100644 --- a/yaffs_yaffs2.c +++ b/yaffs_yaffs2.c @@ -836,7 +836,7 @@ int yaffs2_handle_hole(struct yaffs_obj *obj, loff_t new_size) * of hole marker. */ loff_t old_file_size; - int increase; + loff_t increase; int small_hole; int result = YAFFS_OK; struct yaffs_dev *dev = NULL;