direct/python/yaffs_importer.py

   1 import os
   2 from yaffsfs import *
   3 import sys
   4 import ctypes
   5
   6
   7 #dir_in_snapshot=[]
   8 #files_in_snapshot=[]
   9 #symlinks_in_snapshot=[]
  10 #unknown_in_snapshot=[]
  11 #is_mount_in_snapshot=[]
  12 def check_for_yaffs_errors(output):
  13     if output<0:
  14         ##error has happened
  15         error=ctypes.c_int()
  16         error=yaffs_get_error()
  17         debug_message("error######################################",0)
  18         debug_message(("error code", error),  0)
  19
  20 def debug_message(message, debug_level):
  21     """notew that debug level 0 will always be printed"""
  22     """level 0 error messages"""
  23     """level 1 basic tasks are shown(creating, deleating,ect)"""
  24     """level 2 all process are shown"""
  25     """level 3 shows minor tasks such as join_paths, ect"""
  26     """level 4 is used for bug hunting and shows each step in detail"""
  27     if current_debug_level>=debug_level:
  28         print message
  29
  30 def join_paths(path1, path2):
  31     new_path=path1
  32     if path1[len(path1)-1]=="/"and path2[0]=="/":
  33         new_path+=path2[1:]
  34     elif path1[len(path1)-1]!="/"and path2[0]!="/":
  35         new_path+="/"
  36         new_path+=path2
  37     else:
  38         new_path+=path2
  39
  40     debug_message(("adding path ", path1, " to ",  path2, " resulting path: ", new_path), 3)
  41     return new_path
  42
  43 def subtract_paths(path1, path2):
  44     if len(path1)>len(path2):
  45         debug_message("the two path1 is longer than path2 and can therefore be subtracted.", 4)
  46         ##if the two paths are diretly subtractable
  47         if path1[0:len (path2)-1]==path2:
  48             debug_message("the two paths are direcly subtractable", 4)
  49             new_path=path1[len(path2):]
  50         elif path1[1:len (path2)-1]==path2:
  51             debug_message("the path1 has one more charecter at the begining. assuming that the first chareter is a slash", 4)##fix this assumption.
  52             new_path=path1[len(path2)+1:]
  53         elif path1[1:len (path2)]==path2[1:]:
  54             debug_message("the path2 has one more charecter at the begining. assuming that the first chareter is a slash", 4)##fix this assumption.
  55
  56             new_path=path1[len(path2)+1:]
  57         else :
  58             debug_message("error:could not subtract paths", 0)
  59             debug_message( ("paths do not match:"+ path1+ "  "+path2), 0)
  60             return 0
  61     else :
  62         debug_message( ("cannot subtract path2(:", path2, ") from path1(", path1, ")because path 2 is too long"), 0)
  63
  64         return 0
  65     debug_message(("subtracting paths ", path2, " from ",  path1, " resulting path: ", new_path), 3)
  66     return new_path
  67
  68 def create_file(file):
  69     debug_message( "\n \n \n", 2)
  70     file_path= join_paths(yaffs_root_dir_path, file["path"][len(path):])
  71     debug_message( ("creating file:", file_path), 2)
  72     debug_message (("mode", file["mode"]), 2)
  73     debug_message("opening file",2)
  74 #    yaffs_ls(file["path"])
  75
  76     ##if there is already a file in yaffs then remove the file . this is to prevent yaffs from opening a nd writing to a read only file
  77     if yaffs_access(file_path, 0)==0:##the 0 means does it exist.
  78         debug_message ("file already exists in yaffs", 2)
  79         output=yaffs_unlink(file_path)
  80         debug_message(("unlinking", file_path, output), 2)
  81         check_for_yaffs_errors(output)
  82
  83     current_handle=yaffs_open(file_path, yaffs_O_CREAT | yaffs_O_TRUNC| yaffs_O_RDWR, yaffs_S_IREAD | yaffs_S_IWRITE)  ##opens a file with mode set to write
  84     debug_message(("current_handle", current_handle), 2)
  85     data_file=open(file["path"], "r")
  86     output=yaffs_lseek(current_handle, 0, 0)
  87     if output==-1:
  88         debug_message("error with yaffs lseeking", 2)
  89
  90         check_for_yaffs_errors(output)
  91     data_to_be_written= data_file.read()
  92
  93
  94     length_of_file=len(data_to_be_written)
  95     debug_message (("length of data to be written",length_of_file), 2)
  96     output=yaffs_write(current_handle,data_to_be_written , length_of_file)
  97     if output>=0:
  98         debug_message(( "writing file:", output), 2)
  99     else :
 100         debug_message(( "error writing file:", output), 0)
 101         check_for_yaffs_errors(output)
 102     output=yaffs_ftruncate(current_handle, length_of_file)
 103     if output>=0:
 104         debug_message(( "truncating file:", output), 2)
 105     else :
 106         debug_message(( "error truncating file:", output), 0)
 107         check_for_yaffs_errors(output)
 108     output=yaffs_close(current_handle)
 109     if output>=0:
 110         debug_message(( "closing file:", output), 2)
 111     else :
 112         debug_message(( "error closing file:", output), 0)
 113         check_for_yaffs_errors(output)
 114     ##changes the mode of the yaffs file to be the same as the scanned file
 115     yaffs_chmod(file_path, file["mode"]);
 116     if output>=0:
 117         debug_message(( "chmoding file:", output), 2)
 118     else :
 119         debug_message(( "error chmoding file:", output), 0)
 120         check_for_yaffs_errors(output)
 121
 122
 123
 124 def create_dir(dir, scanned_path, yaffs_path):
 125     debug_message( "\n \n \n", 2)
 126     absolute_dir_path=join_paths(yaffs_path, subtract_paths(dir["path"],scanned_path))
 127     debug_message( ("creating dir:", absolute_dir_path), 2)
 128     debug_message (("mode", dir["mode"]), 2)
 129
 130
 131     ##if there is already a dir in yaffs then remove the dir . this is to clean the yaffs folder if it already exists.
 132     if yaffs_access(absolute_dir_path, 0)==0:##the 0 means does it exist.
 133         debug_message ("folder already exists in yaffs", 2)
 134         output=yaffs_unlink(absolute_dir_path)
 135         debug_message(("unlinking", absolute_dir_path, output), 2)
 136         check_for_yaffs_errors(output)
 137
 138     output=yaffs_mkdir(absolute_dir_path, dir["mode"] )
 139     if output>=0:
 140         debug_message(( "creating dir:", output), 2)
 141     else :
 142         debug_message(( "error creating dir:", output), 0)
 143         check_for_yaffs_errors(output)
 144
 145
 146
 147
 148 def remove_file_from_path(path):
 149     slash_id=[]
 150     for i in range(0, len(path)):
 151         if path[i]=="/":
 152             slash_id.append(i)
 153     new_path=path[:slash_id[len(slash_id)-1]]
 154     debug_message( ("removed file from path", new_path), 2)
 155     return new_path
 156 def is_dir_hidden(dir):
 157     """this code tests if a directory is hidden (has a ./<name> format) and returns true if it is hidden"""
 158     slash_id=[]
 159     for i in range(0, len(dir)):
 160         if dir[i]=="/":
 161             slash_id.append(i)
 162
 163     if dir[slash_id[len(slash_id)-1]+1]==".":
 164         return True
 165     else :
 166         return False
 167
 168 def scan_dir(path, search_hidden_directories=True, ):
 169     """this function scans all of the files and directories in a directory. The function then calls its self on any of the directories that it found. this causes it to build up a tree of all the files and directories """
 170     files_in_snapshot=[]
 171     symlinks_in_snapshot=[]
 172     dir_in_snapshot=[]
 173     dir_in_current_dir=[]
 174     unknown_in_snapshot=[]
 175     if os.path.exists(path)==False:
 176         debug_message ("error#############################",0)
 177         debug_message (("path:", path, "  doesnot exist"), 0)
 178         return 0
 179     dir_snapshot=os.listdir(path)
 180     for i in range(0, len(dir_snapshot)):
 181
 182         current_snapshot=os.path.join(path, dir_snapshot[i])
 183         debug_message (("current snapshot:", current_snapshot), 2)
 184         isDir=os.path.isdir(current_snapshot)
 185         isFile=os.path.isfile(current_snapshot)
 186         isLink=os.path.islink(current_snapshot)
 187         isMount=os.path.ismount(current_snapshot)
 188
 189         stat=os.lstat(current_snapshot)
 190
 191         ##note the order of these if and elif statemens is importaint since a file can be symbloic link and a file
 192         if isDir:
 193             if search_hidden_directories==True or (is_dir_hidden(current_snapshot) ==False or search_hidden_directories==True ) :
 194 #                st_mode ##mode of the folder read/write ect
 195                 dir_in_snapshot.append({"path":current_snapshot, "mode":stat.st_mode})
 196                 dir_in_current_dir.append(current_snapshot)
 197             else :
 198                 debug_message( ("file is hidden so it is ingored", current_snapshot,), 1)
 199         elif  isLink:
 200
 201             ##for some reason the os.readlink only gives the target link realative to the directory which the symbloic link is in. change this into a absolute path
 202             x=current_snapshot
 203             x=remove_file_from_path(x)
 204             target=join_paths(x,os.readlink(current_snapshot) )
 205             symlinks_in_snapshot.append({"path":current_snapshot, "target":target})
 206         elif isFile:
 207
 208 #            stat.st_ino ##inode number
 209 #            st_nlink ##number of hard links to this file
 210 #            st_size ##size of file
 211             files_in_snapshot.append({"path":current_snapshot, "inode": stat.st_ino, "size":stat.st_size, "num_of_hardlinks":stat.st_nlink, "mode":stat.st_mode})
 212
 213 #        elif isMount:
 214 #            is_mount_in_snapshot.append(current_snapshot)
 215         else:
 216             unknown_in_snapshot.append(current_snapshot)
 217
 218     for i in range(0, len(dir_in_current_dir)):
 219         scan_dir(dir_in_current_dir[i])
 220     return (files_in_snapshot, dir_in_snapshot, symlinks_in_snapshot, unknown_in_snapshot)
 221 ##
 222 ##def print_scanned_dir_list():
 223 ##    global files_in_snapshot
 224 ##    global symlinks_in_snapshot
 225 ##    print( "scanning dir", 2)
 226 ##
 227 ##
 228 ##    for i in range(0, len(files_in_snapshot)):
 229 ##        if files_in_snapshot[i]["num_of_hardlinks"]>1:
 230 ##            print "inode",files_in_snapshot[i]["inode"],"size",files_in_snapshot[i]["size"],"path:", files_in_snapshot[i]["path"], "    num of hard links",  files_in_snapshot[i]["num_of_hardlinks"]
 231 ##
 232 ##        else :
 233 ##            print "inode",files_in_snapshot[i]["inode"],"size",files_in_snapshot[i]["size"],"path:", files_in_snapshot[i]["path"]
 234 ###        current_open_file=open(files_in_snapshot[i], "r")
 235 ###        #current_open_file.f.read(3)
 236 ###        lines_in_file=current_open_file.readlines()
 237 ###        #use for loop to write code into yaffs file
 238 ###        print "number of line of code:", len(lines_in_file)
 239 ###    print current_open_file
 240 ##    for i in range(0, len(symlinks_in_snapshot)):
 241 ##        print "symlinks in snapshot:", symlinks_in_snapshot[i]
 242 ##    for i in range(0, len(dir_in_snapshot)):
 243 ##        print "directories in snapshot:", dir_in_snapshot[i]
 244 ##    for i in range(0, len(unknown_in_snapshot)):
 245 ##        print "unknown objects in snapshot:", unknown_in_snapshot[i]
 246 ##
 247
 248
 249 def copy_scanned_files_into_yaffs(files_in_snapshot,dir_in_snapshot,  symlinks_in_snapshot, unknown_in_snapshot,   path, yaffs_root_dir_path ):
 250 #files_in_snapshot, dir_in_snapshot, symlinks_in_snapshot, unknown_in_snapshot
 251 #########################################copy directories into yaffs so the files can be created in these directories
 252     debug_message("making directories in yaffs", 1)
 253     for i in range(0, len(dir_in_snapshot)):
 254
 255
 256         dir_path=join_paths(yaffs_root_dir_path,subtract_paths(dir_in_snapshot[i]["path"], path) )
 257         create_dir(dir_in_snapshot[i], path, yaffs_root_dir_path)
 258 #        output=yaffs_mkdir(dir_path,dir_in_snapshot[i]["mode"] )
 259 #        debug_message(("made directory:", dir_path,   "  output", output), 1)
 260 #        debug_message(("mode" ,dir_in_snapshot[i]["mode"]), 2)
 261
 262
 263
 264 #########################################copy file into yaffs
 265     debug_message("copying scanned files into yaffs", 1)
 266     list=[]
 267     inode_blacklist=[]
 268
 269     debug_message("files to be copyied into yaffs", 2)
 270     for a in range(0, len(files_in_snapshot)):
 271         debug_message(files_in_snapshot[a], 2)
 272     debug_message("\n\n\n", 2)
 273     for i in range(0, len(files_in_snapshot)):
 274         list=[]
 275         if files_in_snapshot[i]["num_of_hardlinks"]>1 and files_in_snapshot[i]["inode"] not in inode_blacklist :
 276             debug_message("found a hard link", 2)
 277             debug_message(("inode",files_in_snapshot[i]["inode"],"size",files_in_snapshot[i]["size"],"path:", files_in_snapshot[i]["path"], "    num of hard links",  files_in_snapshot[i]["num_of_hardlinks"] ), 2)
 278             for a in range(0, len(files_in_snapshot) ) :
 279                 if files_in_snapshot[a]["inode"] ==files_in_snapshot[i]["inode"]  :
 280                     ##and os.path.isfile(files_in_snapshot[i])
 281                     debug_message(("found this file which matches inode",files_in_snapshot[a]), 2)
 282                     list.append(files_in_snapshot[a])
 283                     debug_message(("length of list", len(list)), 2)
 284                 if len(list)==files_in_snapshot[i]["num_of_hardlinks"]:
 285                     break
 286             for a in range(0, len(list)):
 287                 debug_message(list[a], 2)
 288             ##add inode to blacklist. all of the indoes in the list should be the same.
 289             inode_blacklist.append(list[0]["inode"])
 290             ##create a file from the first hardlink.
 291             create_file(list[0])
 292             target_path=yaffs_root_dir_path+list[0]["path"][len(path):]
 293             for i in range(1, len(list)):
 294                 debug_message("creating_symlink", 2)
 295                 debug_message(("target path", target_path), 2)
 296                 hardlink_path=yaffs_root_dir_path+list[i]["path"][len(path):]
 297                 debug_message(("hardlink path", hardlink_path), 2)
 298                 output=yaffs_link(target_path,hardlink_path)
 299                 debug_message(("creating hardlink:", list[i]["path"], "output:", output), 1)
 300         elif files_in_snapshot[i]["inode"] not in inode_blacklist :
 301             create_file(files_in_snapshot[i])
 302
 303
 304 ############################copy symlinks into yaffs
 305
 306     for i in range(0, len(symlinks_in_snapshot)):
 307         debug_message(("symlinks in snapshot:", symlinks_in_snapshot[i]), 2)
 308         target_path=join_paths(yaffs_root_dir_path, subtract_paths(symlinks_in_snapshot[i]["target"],  path))
 309         new_path=join_paths(yaffs_root_dir_path, subtract_paths(symlinks_in_snapshot[i]["path"], path))
 310         output=yaffs_symlink(target_path, new_path)
 311         debug_message(("created symlink",new_path , " > ", target_path, "  output:", output), 1)
 312         ##yaffs_symlink(const YCHAR *oldpath, const YCHAR *newpath);
 313
 314
 315     for i in range(0, len(unknown_in_snapshot)):
 316         debug_message( ("unknown object in snapshot:", unknown_in_snapshot[i]), 0)
 317
 318
 319 def import_into_yaffs(file_path, yaffs_path="/yaffs2/", debug_level=1,  copy_hidden_dir=True,new_yaffs_trace_val=0 ):
 320 #    global current_debug_level
 321 #    global search_hidden_directories
 322 #    global yaffs_root_dir_path
 323 #    global path
 324
 325 #    current_debug_level=debug_level
 326 #    search_hidden_directories=copy_hidden_dir
 327 #    yaffs_root_dir_path=yaffs_path
 328 #    path=file_path
 329     old_yaffs_trace_val=yaffs_get_trace()
 330     yaffs_set_trace(new_yaffs_trace_val)
 331
 332     data=scan_dir(file_path, copy_hidden_dir)
 333     copy_scanned_files_into_yaffs(data[0], data[1], data[2], data[3],file_path,  yaffs_path)
 334
 335     yaffs_set_trace(old_yaffs_trace_val)
 336
 337
 338 if __name__=="__main__":
 339     yaffs_StartUp()
 340     yaffs_mount("/yaffs2/")
 341     yaffs_set_trace(0)
 342 #    absolute_path = os.path.abspath(os.path.curdir)
 343     #print "absolute path:", absolute_path
 344     current_debug_level=1
 345     search_hidden_directories=True
 346     yaffs_root_dir_path="/yaffs2/"
 347     #print sys.argv
 348     path=sys.argv[1]
 349     for i in range(2, len(sys.argv)):
 350         if sys.argv[i]=="-d":
 351             current_debug_level=int( sys.argv[i+1])
 352         if sys.argv[i]=="-ignore_hidden_directories":
 353             search_hidden_directories=False
 354 #
 355 #
 356 #    path="/home/timothy/work/yaffs/git/yaffs2"
 357 #    path="/home/timothy/my_stuff/old_laptop/timothy/programming_lejos/"
 358
 359
 360     import_into_yaffs(path, yaffs_root_dir_path, current_debug_level,  search_hidden_directories, 0 )
 361 #    scan_dir(path)
 362 #    copy_scanned_files_into_yaffs()
 363     #print_scanned_dir_list()
 364
 365     print"unmounting yaffs:", yaffs_unmount("/yaffs2/")