Mercurial > agora
annotate apps/bundle/tasks.py @ 210:95edf106bdef
Create BundleVersion when uploading text bundles
Forgot to add that earlier
author | dellsystem <ilostwaldo@gmail.com> |
---|---|
date | Tue, 19 Feb 2013 01:53:59 -0500 |
parents | 4033ebe1867f |
children |
rev | line source |
---|---|
166
a05e547449de
Oops, move 'from __future__ import' statement to top of the file
Jordi Gutiérrez Hermoso <jordigh@octave.org>
parents:
165
diff
changeset
|
1 from __future__ import with_statement |
a05e547449de
Oops, move 'from __future__ import' statement to top of the file
Jordi Gutiérrez Hermoso <jordigh@octave.org>
parents:
165
diff
changeset
|
2 |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
3 import mimetypes |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
4 import os |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
5 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
6 import archive |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
7 import magic |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
8 from celery import task |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
9 |
209
4033ebe1867f
Add ability to download files
dellsystem <ilostwaldo@gmail.com>
parents:
184
diff
changeset
|
10 from apps.bundle.models import Bundle, BundleFile, BundleVersion |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
11 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
12 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
13 mimetypes.add_type('application/x-gzip', '.tgz') |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
14 mimetypes.add_type('application/x-bzip2', '.tz2') |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
15 archive_extensions = ('.zip', '.tgz', '.tar', '.tz2') |
179
76abe6d681ea
Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents:
177
diff
changeset
|
16 text_mimetypes = ( |
76abe6d681ea
Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents:
177
diff
changeset
|
17 'application/xml', |
76abe6d681ea
Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents:
177
diff
changeset
|
18 ) |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
19 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
20 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
21 def process_files_in_dir(bundle, dir_name, parent_dir): |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
22 """ |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
23 Go through all the subdirectories and files (in that order) in this |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
24 directory. |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
25 """ |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
26 dir_contents = [os.path.join(dir_name, f) for f in os.listdir(dir_name)] |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
27 dirs = filter(os.path.isdir, dir_contents) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
28 files = filter(os.path.isfile, dir_contents) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
29 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
30 for file_path in sorted(dirs) + sorted(files): |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
31 filename = os.path.basename(file_path) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
32 full_path = file_path[len(bundle.get_temp_path()) + 1:] |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
33 bundle_file = BundleFile(bundle=bundle, name=filename, |
177
86129d185ddb
Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents:
166
diff
changeset
|
34 parent=parent_dir, full_path=full_path, |
86129d185ddb
Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents:
166
diff
changeset
|
35 version=bundle.latest_version) |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
36 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
37 if file_path in files: |
184
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
38 is_desc = False |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
39 bundle_file.is_dir = False |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
40 bundle_file.file_size = os.path.getsize(file_path) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
41 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
42 # Only highlight the file contents if it's plain text |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
43 mime_type = magic.from_file(file_path, mime=True) |
179
76abe6d681ea
Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents:
177
diff
changeset
|
44 if mime_type.startswith('text/') or mime_type in text_mimetypes: |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
45 with open(file_path, 'rt') as file: |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
46 # Store the contents of the file in the code field |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
47 bundle_file.save_file_contents(file) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
48 |
184
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
49 # DESCRIPTION file should be at most 1 level deep |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
50 single_parent = (parent_dir is not None or |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
51 '/' not in parent_dir) |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
52 is_desc = single_parent and filename == 'DESCRIPTION' |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
53 |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
54 # Check if this is the description file (if no description exists) |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
55 if bundle.octave_format and bundle.description == '' and is_desc: |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
56 bundle.description_file = bundle_file |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
57 bundle.save() |
b711f0087709
Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents:
179
diff
changeset
|
58 |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
59 bundle_file.save() |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
60 else: |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
61 # It's a directory - call this function on it recursively |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
62 bundle_file.is_dir = True |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
63 bundle_file.save() |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
64 process_files_in_dir(bundle, file_path, bundle_file) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
65 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
66 @task() |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
67 def handle_bundle_upload(bundle_id): |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
68 bundle = Bundle.objects.get(pk=bundle_id) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
69 file = bundle.get_temp_path() |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
70 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
71 # Figure out the most likely mime-type |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
72 mime_type = magic.from_file(file, mime=True) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
73 extension = mimetypes.guess_extension(mime_type) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
74 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
75 if extension in archive_extensions: |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
76 new_path = file + extension |
210
95edf106bdef
Create BundleVersion when uploading text bundles
dellsystem <ilostwaldo@gmail.com>
parents:
209
diff
changeset
|
77 # Save the new file name (needed for the BundleVersion) |
95edf106bdef
Create BundleVersion when uploading text bundles
dellsystem <ilostwaldo@gmail.com>
parents:
209
diff
changeset
|
78 |
95edf106bdef
Create BundleVersion when uploading text bundles
dellsystem <ilostwaldo@gmail.com>
parents:
209
diff
changeset
|
79 new_file_name = os.path.basename(new_path) |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
80 # Treat it as an archive. Rename it to that, then extract |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
81 os.rename(file, new_path) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
82 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
83 try: |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
84 # Extract it to a directory, same path as the filename |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
85 archive.extract(new_path, to_path=file) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
86 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
87 # Now go through the extracted files, make BundleFiles from them |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
88 process_files_in_dir(bundle, file, None) |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
89 except archive.ArchiveException: |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
90 pass |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
91 elif mime_type.startswith('text/'): |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
92 # Should be a plain text file - create a CodeFile for it |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
93 bundle_file = BundleFile(bundle=bundle, name=bundle.file_name, |
177
86129d185ddb
Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents:
166
diff
changeset
|
94 full_path=bundle.file_name, file_size=os.path.getsize(file), |
86129d185ddb
Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents:
166
diff
changeset
|
95 version=bundle.latest_version) |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
96 bundle_file.save_file_contents(open(file, 'rt'), |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
97 original_filename=bundle.file_name) |
210
95edf106bdef
Create BundleVersion when uploading text bundles
dellsystem <ilostwaldo@gmail.com>
parents:
209
diff
changeset
|
98 new_file_name = os.path.basename(file) |
95edf106bdef
Create BundleVersion when uploading text bundles
dellsystem <ilostwaldo@gmail.com>
parents:
209
diff
changeset
|
99 |
95edf106bdef
Create BundleVersion when uploading text bundles
dellsystem <ilostwaldo@gmail.com>
parents:
209
diff
changeset
|
100 # Create the new BundleVersion |
95edf106bdef
Create BundleVersion when uploading text bundles
dellsystem <ilostwaldo@gmail.com>
parents:
209
diff
changeset
|
101 BundleVersion.objects.create(bundle=bundle, file_name=new_file_name, |
95edf106bdef
Create BundleVersion when uploading text bundles
dellsystem <ilostwaldo@gmail.com>
parents:
209
diff
changeset
|
102 version=bundle.latest_version) |
151
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
103 |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
104 bundle.done_uploading = True |
c7be7def8b57
Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff
changeset
|
105 bundle.save() |