Mercurial > agora

annotate apps/bundle/tasks.py @ 184:b711f0087709

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Use DESCRIPTION file for bundles (SCHEMA CHANGE) * Added two new fields to the Bundle model: * octave_format, which allows users to specify if their bundle has been formatted according to octave packaging standards or not * description_file, which points to a file named DESCRIPTION in the root directory (or the next top-level directory), if the octave_format checkbox is ticked and if one exists * Fixed the uploader field for form by making it a hidden input and preventing hidden inputs from showing up entirely
author dellsystem <ilostwaldo@gmail.com>
date Sat, 27 Oct 2012 15:58:08 -0400
parents 76abe6d681ea
children 4033ebe1867f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
166
a05e547449de Oops, move 'from __future__ import' statement to top of the file
Jordi Gutiérrez Hermoso <jordigh@octave.org>
parents: 165
diff changeset
1 from __future__ import with_statement
a05e547449de Oops, move 'from __future__ import' statement to top of the file
Jordi Gutiérrez Hermoso <jordigh@octave.org>
parents: 165
diff changeset
2
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
3 import mimetypes
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
4 import os
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
5
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
6 import archive
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
7 import magic
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
8 from celery import task
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
9
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
10 from apps.bundle.models import Bundle, BundleFile
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
11
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
12
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
13 mimetypes.add_type('application/x-gzip', '.tgz')
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
14 mimetypes.add_type('application/x-bzip2', '.tz2')
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
15 archive_extensions = ('.zip', '.tgz', '.tar', '.tz2')
179
76abe6d681ea Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents: 177
diff changeset
16 text_mimetypes = (
76abe6d681ea Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents: 177
diff changeset
17 'application/xml',
76abe6d681ea Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents: 177
diff changeset
18 )
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
19
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
20
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
21 def process_files_in_dir(bundle, dir_name, parent_dir):
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
22 """
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
23 Go through all the subdirectories and files (in that order) in this
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
24 directory.
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
25 """
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
26 dir_contents = [os.path.join(dir_name, f) for f in os.listdir(dir_name)]
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
27 dirs = filter(os.path.isdir, dir_contents)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
28 files = filter(os.path.isfile, dir_contents)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
29
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
30 for file_path in sorted(dirs) + sorted(files):
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
31 filename = os.path.basename(file_path)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
32 full_path = file_path[len(bundle.get_temp_path()) + 1:]
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
33 bundle_file = BundleFile(bundle=bundle, name=filename,
177
86129d185ddb Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents: 166
diff changeset
34 parent=parent_dir, full_path=full_path,
86129d185ddb Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents: 166
diff changeset
35 version=bundle.latest_version)
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
36
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
37 if file_path in files:
184
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
38 is_desc = False
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
39 bundle_file.is_dir = False
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
40 bundle_file.file_size = os.path.getsize(file_path)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
41
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
42 # Only highlight the file contents if it's plain text
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
43 mime_type = magic.from_file(file_path, mime=True)
179
76abe6d681ea Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents: 177
diff changeset
44 if mime_type.startswith('text/') or mime_type in text_mimetypes:
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
45 with open(file_path, 'rt') as file:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
46 # Store the contents of the file in the code field
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
47 bundle_file.save_file_contents(file)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
48
184
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
49 # DESCRIPTION file should be at most 1 level deep
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
50 single_parent = (parent_dir is not None or
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
51 '/' not in parent_dir)
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
52 is_desc = single_parent and filename == 'DESCRIPTION'
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
53
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
54 # Check if this is the description file (if no description exists)
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
55 if bundle.octave_format and bundle.description == '' and is_desc:
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
56 bundle.description_file = bundle_file
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
57 bundle.save()
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
58
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
59 bundle_file.save()
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
60 else:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
61 # It's a directory - call this function on it recursively
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
62 bundle_file.is_dir = True
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
63 bundle_file.save()
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
64 process_files_in_dir(bundle, file_path, bundle_file)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
65
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
66 @task()
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
67 def handle_bundle_upload(bundle_id):
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
68 bundle = Bundle.objects.get(pk=bundle_id)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
69 file = bundle.get_temp_path()
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
70
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
71 # Figure out the most likely mime-type
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
72 mime_type = magic.from_file(file, mime=True)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
73 extension = mimetypes.guess_extension(mime_type)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
74
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
75 print "mime type: %s" % mime_type
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
76 print "extension: %s" % extension
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
77
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
78 if extension in archive_extensions:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
79 new_path = file + extension
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
80 # Treat it as an archive. Rename it to that, then extract
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
81 os.rename(file, new_path)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
82
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
83 try:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
84 # Extract it to a directory, same path as the filename
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
85 archive.extract(new_path, to_path=file)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
86
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
87 # Now go through the extracted files, make BundleFiles from them
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
88 process_files_in_dir(bundle, file, None)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
89 except archive.ArchiveException:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
90 print "Archive exception"
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
91 pass
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
92 elif mime_type.startswith('text/'):
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
93 # Should be a plain text file - create a CodeFile for it
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
94 bundle_file = BundleFile(bundle=bundle, name=bundle.file_name,
177
86129d185ddb Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents: 166
diff changeset
95 full_path=bundle.file_name, file_size=os.path.getsize(file),
86129d185ddb Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents: 166
diff changeset
96 version=bundle.latest_version)
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
97 bundle_file.save_file_contents(open(file, 'rt'),
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
98 original_filename=bundle.file_name)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
99
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
100 print "Done uploading!"
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
101 bundle.done_uploading = True
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
102 bundle.save()