direct/python/yaffs_importer.py

   1 ##
   2 ## YAFFS: Yet Another Flash File System. A NAND-flash specific file system.
   3 ##
   4 ## Copyright (C) 2002-2010 Aleph One Ltd.
   5 ##   for Toby Churchill Ltd and Brightstar Engineering
   6 ##
   7 ## Created by Timothy Manning <timothy@yaffs.net>
   8 ##
   9 ## This program is free software; you can redistribute it and/or modify
  10 ## it under the terms of the GNU General Public License version 2 as
  11 ## published by the Free Software Foundation.
  12 ##
  13
  14 import os
  15 from yaffsfs import *
  16 import sys
  17 import ctypes
  18
  19
  20 dir_in_snapshot=[]
  21 files_in_snapshot=[]
  22 symlinks_in_snapshot=[]
  23 unknown_in_snapshot=[]
  24 is_mount_in_snapshot=[]
  25 def check_for_yaffs_errors(output):
  26     if output<0:
  27         ##error has happened
  28         error=ctypes.c_int()
  29         error=yaffs_get_error()
  30         debug_message("error######################################",0)
  31         debug_message(("error code", error),  0)
  32         error_message=ctypes.c_char_p()
  33         error_message.value=yaffs_error_to_str(error);
  34         print "error message", error_message.value
  35
  36
  37 def debug_message(message, debug_level):
  38     """note: that debug level 0 will always be printed unless debug_level is set to -1"""
  39     """level 0 error messages"""
  40     """level 1 basic tasks are shown(creating, deleating,ect)"""
  41     """level 2 all process are shown"""
  42     """level 3 shows minor tasks such as join_paths, ect"""
  43     """level 4 is used for bug hunting and shows each step in detail"""
  44     if current_debug_level>=debug_level:
  45 #        for i in range(0, len(message)):
  46 #            print message,
  47 #        print"\n \n \n"
  48         print message
  49 def join_paths(path1, path2):
  50     new_path=path1
  51     if path1[len(path1)-1]=="/"and path2[0]=="/":
  52         new_path+=path2[1:]
  53     elif path1[len(path1)-1]!="/"and path2[0]!="/":
  54         new_path+="/"
  55         new_path+=path2
  56     else:
  57         new_path+=path2
  58
  59     debug_message(("adding path ", path1, " to ",  path2, " resulting path: ", new_path), 3)
  60     return new_path
  61
  62 def subtract_paths(path1, path2):
  63     if len(path1)>len(path2):
  64         if path1[len(path1)-1]!="/":
  65             path1 +="/"
  66         if path2[len(path2)-1]!="/":
  67             path2 += "/"
  68         debug_message("the two path1 is longer than path2 and can therefore be subtracted.", 4)
  69         ##if the two paths are diretly subtractable
  70         if path1[0:len (path2)-1]==path2:
  71             debug_message("the two paths are direcly subtractable", 4)
  72             new_path=path1[len(path2):]
  73         elif path1[1:len (path2)-1]==path2:
  74             debug_message("the path1 has one more charecter at the begining. assuming that the first chareter is a slash", 4)##fix this assumption.
  75             new_path=path1[len(path2)+1:]
  76         elif path1[1:len (path2)]==path2[1:]:
  77             debug_message("the path2 has one more charecter at the begining. assuming that the first chareter is a slash", 4)##fix this assumption.
  78
  79             new_path=path1[len(path2)-1:]
  80         else :
  81             debug_message("error:could not subtract paths", 0)
  82             debug_message( ("paths do not match:"+ path1+ "  "+path2), 0)
  83             return 0
  84     else :
  85         debug_message( ("cannot subtract path2(:", path2, ") from path1(", path1, ")because path 2 is too long"), 0)
  86
  87         return 0
  88     debug_message(("subtracting paths ", path2, " from ",  path1, " resulting path: ", new_path), 3)
  89     return new_path
  90
  91 def create_file(file):
  92 #    freespace=ctypes.c_longlong
  93 #    freespace.value=yaffs_freespace(yaffs_root_dir_path)
  94 #    print "yaffs free space:", freespace.value
  95     debug_message( "\n \n \n", 2)
  96     file_path= join_paths(yaffs_root_dir_path, file["path"][len(path):])
  97     debug_message( ("creating file:", file_path), 2)
  98     debug_message (("mode", file["mode"]), 2)
  99     debug_message("opening file",2)
 100 #    yaffs_ls(file["path"])
 101
 102     ##if there is already a file in yaffs then remove the file . this is to prevent yaffs from opening and writing to a read only file
 103     if yaffs_access(file_path, 0)==0:##the 0 means does it exist.
 104         debug_message ("file already exists in yaffs", 2)
 105         output=yaffs_unlink(file_path)
 106         debug_message(("unlinking", file_path, output), 2)
 107         check_for_yaffs_errors(output)
 108
 109     current_handle=yaffs_open(file_path, yaffs_O_CREAT | yaffs_O_TRUNC| yaffs_O_RDWR, yaffs_S_IREAD | yaffs_S_IWRITE)  ##opens a file with mode set to write
 110     debug_message(("current_handle", current_handle), 2)
 111     data_file=open(file["path"], "r")
 112     output=yaffs_lseek(current_handle, 0, 0)
 113     if output==-1:
 114         ##if there is no more space in the emfile then this is where it will show up.
 115         freespace=ctypes.c_longlong
 116         freespace.value=yaffs_freespace(yaffs_root_dir_path)
 117         print "yaffs free space:", freespace.value
 118
 119         if freespace.value==0:
 120             #print "yaffs free space:", yaffs_freespace(yaffs_root_dir_path)
 121             print "yaffs is out of space exiting program"
 122            #sys.exit()
 123         debug_message("error with yaffs lseeking", 2)
 124
 125         check_for_yaffs_errors(output)
 126     data_to_be_written= data_file.read()
 127
 128
 129     length_of_file=len(data_to_be_written)
 130     debug_message (("length of data to be written",length_of_file), 3)
 131     output=yaffs_write(current_handle,data_to_be_written , length_of_file)
 132     if output>=0:
 133         debug_message(( "writing to ", file_path," ",  output), 1)
 134     else :
 135         debug_message(( "error writing file:", output), 0)
 136         check_for_yaffs_errors(output)
 137     output=yaffs_ftruncate(current_handle, length_of_file)
 138     if output>=0:
 139         debug_message(( "truncating file:", output), 2)
 140     else :
 141         debug_message(( "error truncating file:", output), 0)
 142         check_for_yaffs_errors(output)
 143     output=yaffs_close(current_handle)
 144     if output>=0:
 145         debug_message(( "closing file:", output), 2)
 146     else :
 147         debug_message(( "error closing file:", output), 0)
 148         check_for_yaffs_errors(output)
 149     ##changes the mode of the yaffs file to be the same as the scanned file
 150     yaffs_chmod(file_path, file["mode"]);
 151     if output>=0:
 152         debug_message(( "chmoding file:", output), 2)
 153     else :
 154         debug_message(( "error chmoding file:", output), 0)
 155         check_for_yaffs_errors(output)
 156
 157
 158
 159 def create_dir(dir, scanned_path, yaffs_path):
 160     debug_message( "\n \n \n", 2)
 161     absolute_dir_path=join_paths(yaffs_path, subtract_paths(dir["path"],scanned_path))
 162     debug_message( ("creating dir:", absolute_dir_path), 2)
 163     debug_message (("mode(in octal", oct(dir["mode"])), 2)
 164
 165        ##this is a bug in yaffs which will not make a dir if there is a slash on the end
 166     if absolute_dir_path[len(absolute_dir_path)-1]=="/":
 167         absolute_dir_path=absolute_dir_path[0:len(absolute_dir_path)-1]
 168         debug_message (("path has slash on the end. removing slash new path is:",absolute_dir_path) , 4)
 169
 170
 171     ##if there is already a dir in yaffs then remove the dir . this is to clean the yaffs folder if it already exists.
 172     ##in yaffs all of the files in the dir to be removed must be empty.
 173     ##need to create a reverse ls to delete all of the files first.
 174 #    if yaffs_access(absolute_dir_path, 0)==0:##the 0 means does it exist.
 175 #        debug_message ("folder already exists in yaffs", 2)
 176 #        output=yaffs_rmdir(absolute_dir_path)
 177 #        debug_message(("unlinking", absolute_dir_path, output), 2)
 178 #        check_for_yaffs_errors(output)
 179
 180
 181
 182     output=yaffs_mkdir(absolute_dir_path, dir["mode"] )
 183     if output>=0:
 184         debug_message(( "created dir:", absolute_dir_path," ", output), 1)
 185     else :
 186         debug_message(( "error creating dir ", absolute_dir_path, " ", output), 0)
 187         check_for_yaffs_errors(output)
 188         if output==17:
 189             printf("the directory already exists")
 190
 191
 192
 193
 194 def remove_file_from_path(path):
 195     slash_id=[]
 196     for i in range(0, len(path)):
 197         if path[i]=="/":
 198             slash_id.append(i)
 199     new_path=path[:slash_id[len(slash_id)-1]]
 200     debug_message( ("removed file from path", new_path), 2)
 201     return new_path
 202
 203 def is_dir_hidden(dir):
 204     """this code tests if a directory is hidden (has a ./<name> format) and returns true if it is hidden"""
 205     slash_id=[]
 206     for i in range(0, len(dir)):
 207         if dir[i]=="/":
 208             slash_id.append(i)
 209
 210     if dir[slash_id[len(slash_id)-1]+1]==".":
 211         return True
 212     else :
 213         return False
 214
 215 def scan_dir(path, search_hidden_directories=True, ):
 216     """this function scans all of the files and directories in a directory. The function then calls its self on any of the directories that it found. this causes it to build up a tree of all the files and directories """
 217     global files_in_snapshot
 218     global symlinks_in_snapshot
 219     global dir_in_snapshot
 220     dir_in_current_dir=[]
 221     global unknown_in_snapshot
 222 #    files_in_snapshot=[]
 223 #    symlinks_in_snapshot=[]
 224 #    dir_in_snapshot=[]
 225 #    dir_in_current_dir=[]
 226 #    unknown_in_snapshot=[]
 227     if os.path.exists(path)==False:
 228         debug_message ("error#############################",0)
 229         debug_message (("path:", path, "  doesnot exist"), 0)
 230         return 0
 231     dir_snapshot=os.listdir(path)
 232     for i in range(0, len(dir_snapshot)):
 233
 234         current_snapshot=os.path.join(path, dir_snapshot[i])
 235         ##debug_message (("current snapshot:", current_snapshot), 2)
 236         isDir=os.path.isdir(current_snapshot)
 237         isFile=os.path.isfile(current_snapshot)
 238         isLink=os.path.islink(current_snapshot)
 239         isMount=os.path.ismount(current_snapshot)
 240
 241         stat=os.lstat(current_snapshot)
 242
 243         ##note the order of these if and elif statemens is importaint since a file can be symbloic link and a file
 244         if isDir:
 245             if search_hidden_directories==True or (is_dir_hidden(current_snapshot) ==False or search_hidden_directories==True ) :
 246 #                st_mode ##mode of the folder read/write ect
 247                 dir_in_snapshot.append({"path":current_snapshot, "mode":stat.st_mode})
 248                 dir_in_current_dir.append(current_snapshot)
 249             else :
 250                 debug_message( ("file is hidden so it is ingored", current_snapshot,), 1)
 251         elif  isLink:
 252
 253             ##for some reason the os.readlink only gives the target link realative to the directory which the symbloic link is in. change this into a absolute path
 254             x=current_snapshot
 255             x=remove_file_from_path(x)
 256             target=join_paths(x,os.readlink(current_snapshot) )
 257             symlinks_in_snapshot.append({"path":current_snapshot, "target":target})
 258         elif isFile:
 259
 260 #            stat.st_ino ##inode number
 261 #            st_nlink ##number of hard links to this file
 262 #            st_size ##size of file
 263             files_in_snapshot.append({"path":current_snapshot, "inode": stat.st_ino, "size":stat.st_size, "num_of_hardlinks":stat.st_nlink, "mode":stat.st_mode})
 264
 265 #        elif isMount:
 266 #            is_mount_in_snapshot.append(current_snapshot)
 267         else:
 268             unknown_in_snapshot.append(current_snapshot)
 269
 270     for i in range(0, len(dir_in_current_dir)):
 271         debug_message(("scanning dir", dir_in_current_dir[i]) , 2)
 272         scan_dir(dir_in_current_dir[i], search_hidden_directories)
 273
 274 #        #debug_message(("data 0", data[0][0]), 2)
 275 #        if len(files)
 276 #        files_in_snapshot.append(data[0][0])
 277 #        dir_in_snapshot.append(data[1][0])
 278 #        symlinks_in_snapshot.append(data[2][0])
 279 #        unknown_in_snapshot.append(data[3][0])
 280     return (files_in_snapshot, dir_in_snapshot, symlinks_in_snapshot, unknown_in_snapshot)
 281 ##
 282 ##def print_scanned_dir_list():
 283 ##    global files_in_snapshot
 284 ##    global symlinks_in_snapshot
 285 ##    print( "scanning dir", 2)
 286 ##
 287 ##
 288 ##    for i in range(0, len(files_in_snapshot)):
 289 ##        if files_in_snapshot[i]["num_of_hardlinks"]>1:
 290 ##            print "inode",files_in_snapshot[i]["inode"],"size",files_in_snapshot[i]["size"],"path:", files_in_snapshot[i]["path"], "    num of hard links",  files_in_snapshot[i]["num_of_hardlinks"]
 291 ##
 292 ##        else :
 293 ##            print "inode",files_in_snapshot[i]["inode"],"size",files_in_snapshot[i]["size"],"path:", files_in_snapshot[i]["path"]
 294 ###        current_open_file=open(files_in_snapshot[i], "r")
 295 ###        #current_open_file.f.read(3)
 296 ###        lines_in_file=current_open_file.readlines()
 297 ###        #use for loop to write code into yaffs file
 298 ###        print "number of line of code:", len(lines_in_file)
 299 ###    print current_open_file
 300 ##    for i in range(0, len(symlinks_in_snapshot)):
 301 ##        print "symlinks in snapshot:", symlinks_in_snapshot[i]
 302 ##    for i in range(0, len(dir_in_snapshot)):
 303 ##        print "directories in snapshot:", dir_in_snapshot[i]
 304 ##    for i in range(0, len(unknown_in_snapshot)):
 305 ##        print "unknown objects in snapshot:", unknown_in_snapshot[i]
 306 ##
 307
 308
 309 def copy_scanned_files_into_yaffs(files_in_snapshot, dir_in_snapshot,  symlinks_in_snapshot, unknown_in_snapshot,   path, yaffs_root_dir_path="/yaffs2/", yaffs_mount_point_path="/yaffs2/" ):
 310 #files_in_snapshot, dir_in_snapshot, symlinks_in_snapshot, unknown_in_snapshot
 311 #########################################copy directories into yaffs so the files can be created in these directories
 312     debug_message("making directories in yaffs", 1)
 313     if yaffs_root_dir_path!=yaffs_mount_point_path:
 314         slash_id=[]
 315         debug_message("making directories to the place in yaffs where the directories will copied to", 2)
 316         root_branch_path=subtract_paths(yaffs_root_dir_path, yaffs_mount_point_path)
 317         for i in range(0, len(root_branch_path)):
 318
 319             if root_branch_path[i]=="/" and i != 0:
 320                slash_id.append(i)
 321         debug_message(("slash_id", slash_id),4)
 322         for i in range(0, len(slash_id)):
 323             create_dir({"path":root_branch_path[:slash_id[i]], "mode": yaffs_S_IREAD | yaffs_S_IWRITE}, "/", yaffs_mount_point_path)
 324
 325     for i in range(0, len(dir_in_snapshot)):
 326         create_dir(dir_in_snapshot[i], path, yaffs_root_dir_path)
 327
 328
 329
 330 #########################################copy file into yaffs
 331     debug_message("copying scanned files into yaffs", 1)
 332     list=[]
 333     inode_blacklist=[]
 334
 335     debug_message("files to be copyied into yaffs", 2)
 336     for a in range(0, len(files_in_snapshot)):
 337         debug_message(files_in_snapshot[a], 2)
 338     debug_message("\n\n\n", 2)
 339     for i in range(0, len(files_in_snapshot)):
 340         list=[]
 341         if files_in_snapshot[i]["num_of_hardlinks"]>1 and files_in_snapshot[i]["inode"] not in inode_blacklist :
 342             debug_message("found a hard link", 2)
 343             debug_message(("inode",files_in_snapshot[i]["inode"],"size",files_in_snapshot[i]["size"],"path:", files_in_snapshot[i]["path"], "    num of hard links",  files_in_snapshot[i]["num_of_hardlinks"] ), 2)
 344             for a in range(0, len(files_in_snapshot) ) :
 345                 if files_in_snapshot[a]["inode"] ==files_in_snapshot[i]["inode"]  :
 346                     ##and os.path.isfile(files_in_snapshot[i])
 347                     debug_message(("found this file which matches inode",files_in_snapshot[a]), 2)
 348                     list.append(files_in_snapshot[a])
 349                     debug_message(("length of list", len(list)), 2)
 350                 if len(list)==files_in_snapshot[i]["num_of_hardlinks"]:
 351                     break
 352             for a in range(0, len(list)):
 353                 debug_message(list[a], 2)
 354             ##add inode to blacklist. all of the indoes in the list should be the same.
 355             inode_blacklist.append(list[0]["inode"])
 356             ##create a file from the first hardlink.
 357             create_file(list[0])
 358             target_path=yaffs_root_dir_path+list[0]["path"][len(path):]
 359             for i in range(1, len(list)):
 360                 debug_message("creating_symlink", 2)
 361                 debug_message(("target path", target_path), 2)
 362                 hardlink_path=yaffs_root_dir_path+list[i]["path"][len(path):]
 363                 debug_message(("hardlink path", hardlink_path), 2)
 364                 output=yaffs_link(target_path,hardlink_path)
 365                 debug_message(("creating hardlink:", list[i]["path"], "output:", output), 1)
 366         elif files_in_snapshot[i]["inode"] not in inode_blacklist :
 367             create_file(files_in_snapshot[i])
 368
 369
 370 ############################copy symlinks into yaffs
 371
 372     for i in range(0, len(symlinks_in_snapshot)):
 373         debug_message(("symlinks in snapshot:", symlinks_in_snapshot[i]), 2)
 374         target_path=join_paths(yaffs_root_dir_path, subtract_paths(symlinks_in_snapshot[i]["target"],  path))
 375         new_path=join_paths(yaffs_root_dir_path, subtract_paths(symlinks_in_snapshot[i]["path"], path))
 376         output=yaffs_symlink(target_path, new_path)
 377         debug_message(("created symlink",new_path , " > ", target_path, "  output:", output), 1)
 378         ##yaffs_symlink(const YCHAR *oldpath, const YCHAR *newpath);
 379
 380
 381     for i in range(0, len(unknown_in_snapshot)):
 382         debug_message( ("unknown object in snapshot:", unknown_in_snapshot[i]), 0)
 383
 384
 385 def import_into_yaffs(file_path, yaffs_path="/yaffs2/", debug_level=1,  copy_hidden_dir=True ,new_yaffs_trace_val=-1 ):
 386 #    global current_debug_level
 387 #    global search_hidden_directories
 388 #    global yaffs_root_dir_path
 389 #    global path
 390
 391 #    current_debug_level=debug_level
 392 #    search_hidden_directories=copy_hidden_dir
 393 #    yaffs_root_dir_path=yaffs_path
 394 #    path=file_path
 395     old_yaffs_trace_val=yaffs_get_trace()
 396     if new_yaffs_trace_val!=-1:
 397         yaffs_set_trace(new_yaffs_trace_val)
 398
 399     data=scan_dir(file_path, copy_hidden_dir)
 400     copy_scanned_files_into_yaffs(data[0], data[1], data[2], data[3],file_path,  yaffs_path)
 401
 402     yaffs_set_trace(old_yaffs_trace_val)
 403
 404
 405 if __name__=="__main__":
 406     yaffs_start_up()
 407     yaffs_mount("/yaffs2/")
 408     #yaffs_set_trace(0)
 409 #    absolute_path = os.path.abspath(os.path.curdir)
 410     #print "absolute path:", absolute_path
 411     current_debug_level=1
 412     search_hidden_directories=True
 413     yaffs_root_dir_path="/yaffs2/"
 414     yaffs_trace=-1
 415     #print sys.argv
 416     path=sys.argv[1]
 417     for i in range(2, len(sys.argv)):
 418         if sys.argv[i]=="-d":
 419             current_debug_level=int( sys.argv[i+1])
 420         if sys.argv[i]=="-ignore_hidden_directories":
 421             search_hidden_directories=False
 422         if sys.argv[i]=="-o":
 423             yaffs_root_dir_path=sys.argv[i+1]
 424         if sys.argv[i]=="-yaffs_trace":
 425             yaffs_trace=int(sys.argv[i+1])
 426 #
 427 #
 428 #    path="/home/timothy/work/yaffs/git/yaffs2"
 429 #    path="/home/timothy/my_stuff/old_laptop/timothy/programming_lejos/"
 430
 431
 432     import_into_yaffs(path, yaffs_root_dir_path, current_debug_level,  search_hidden_directories, yaffs_trace )
 433 #    scan_dir(path)
 434 #    copy_scanned_files_into_yaffs()
 435     #print_scanned_dir_list()
 436
 437     print"unmounting yaffs:", yaffs_unmount("/yaffs2/")