Mercurial > agora

annotate apps/bundle/tasks.py @ 209:4033ebe1867f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add ability to download files This makes use of a new model (BundleVersion) to keep track of the locations on disk of the original uploads for each version. This will require some manual processing to get it working for existing bundles, since the information needed isn't being stored at the moment.
author dellsystem <ilostwaldo@gmail.com>
date Sun, 17 Feb 2013 14:57:39 -0500
parents b711f0087709
children 95edf106bdef
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
166
a05e547449de Oops, move 'from __future__ import' statement to top of the file
Jordi Gutiérrez Hermoso <jordigh@octave.org>
parents: 165
diff changeset
1 from __future__ import with_statement
a05e547449de Oops, move 'from __future__ import' statement to top of the file
Jordi Gutiérrez Hermoso <jordigh@octave.org>
parents: 165
diff changeset
2
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
3 import mimetypes
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
4 import os
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
5
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
6 import archive
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
7 import magic
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
8 from celery import task
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
9
209
4033ebe1867f Add ability to download files
dellsystem <ilostwaldo@gmail.com>
parents: 184
diff changeset
10 from apps.bundle.models import Bundle, BundleFile, BundleVersion
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
11
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
12
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
13 mimetypes.add_type('application/x-gzip', '.tgz')
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
14 mimetypes.add_type('application/x-bzip2', '.tz2')
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
15 archive_extensions = ('.zip', '.tgz', '.tar', '.tz2')
179
76abe6d681ea Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents: 177
diff changeset
16 text_mimetypes = (
76abe6d681ea Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents: 177
diff changeset
17 'application/xml',
76abe6d681ea Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents: 177
diff changeset
18 )
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
19
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
20
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
21 def process_files_in_dir(bundle, dir_name, parent_dir):
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
22 """
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
23 Go through all the subdirectories and files (in that order) in this
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
24 directory.
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
25 """
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
26 dir_contents = [os.path.join(dir_name, f) for f in os.listdir(dir_name)]
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
27 dirs = filter(os.path.isdir, dir_contents)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
28 files = filter(os.path.isfile, dir_contents)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
29
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
30 for file_path in sorted(dirs) + sorted(files):
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
31 filename = os.path.basename(file_path)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
32 full_path = file_path[len(bundle.get_temp_path()) + 1:]
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
33 bundle_file = BundleFile(bundle=bundle, name=filename,
177
86129d185ddb Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents: 166
diff changeset
34 parent=parent_dir, full_path=full_path,
86129d185ddb Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents: 166
diff changeset
35 version=bundle.latest_version)
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
36
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
37 if file_path in files:
184
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
38 is_desc = False
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
39 bundle_file.is_dir = False
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
40 bundle_file.file_size = os.path.getsize(file_path)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
41
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
42 # Only highlight the file contents if it's plain text
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
43 mime_type = magic.from_file(file_path, mime=True)
179
76abe6d681ea Add whitelist for other text-like mimetypes
dellsystem <ilostwaldo@gmail.com>
parents: 177
diff changeset
44 if mime_type.startswith('text/') or mime_type in text_mimetypes:
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
45 with open(file_path, 'rt') as file:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
46 # Store the contents of the file in the code field
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
47 bundle_file.save_file_contents(file)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
48
184
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
49 # DESCRIPTION file should be at most 1 level deep
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
50 single_parent = (parent_dir is not None or
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
51 '/' not in parent_dir)
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
52 is_desc = single_parent and filename == 'DESCRIPTION'
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
53
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
54 # Check if this is the description file (if no description exists)
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
55 if bundle.octave_format and bundle.description == '' and is_desc:
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
56 bundle.description_file = bundle_file
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
57 bundle.save()
b711f0087709 Use DESCRIPTION file for bundles (SCHEMA CHANGE)
dellsystem <ilostwaldo@gmail.com>
parents: 179
diff changeset
58
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
59 bundle_file.save()
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
60 else:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
61 # It's a directory - call this function on it recursively
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
62 bundle_file.is_dir = True
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
63 bundle_file.save()
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
64 process_files_in_dir(bundle, file_path, bundle_file)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
65
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
66 @task()
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
67 def handle_bundle_upload(bundle_id):
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
68 bundle = Bundle.objects.get(pk=bundle_id)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
69 file = bundle.get_temp_path()
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
70
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
71 # Figure out the most likely mime-type
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
72 mime_type = magic.from_file(file, mime=True)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
73 extension = mimetypes.guess_extension(mime_type)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
74
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
75 if extension in archive_extensions:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
76 new_path = file + extension
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
77 # Treat it as an archive. Rename it to that, then extract
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
78 os.rename(file, new_path)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
79
209
4033ebe1867f Add ability to download files
dellsystem <ilostwaldo@gmail.com>
parents: 184
diff changeset
80 # Create the new BundleVersion
4033ebe1867f Add ability to download files
dellsystem <ilostwaldo@gmail.com>
parents: 184
diff changeset
81 new_file_name = os.path.basename(new_path)
4033ebe1867f Add ability to download files
dellsystem <ilostwaldo@gmail.com>
parents: 184
diff changeset
82 BundleVersion.objects.create(bundle=bundle, file_name=new_file_name,
4033ebe1867f Add ability to download files
dellsystem <ilostwaldo@gmail.com>
parents: 184
diff changeset
83 version=bundle.latest_version)
4033ebe1867f Add ability to download files
dellsystem <ilostwaldo@gmail.com>
parents: 184
diff changeset
84
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
85 try:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
86 # Extract it to a directory, same path as the filename
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
87 archive.extract(new_path, to_path=file)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
88
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
89 # Now go through the extracted files, make BundleFiles from them
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
90 process_files_in_dir(bundle, file, None)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
91 except archive.ArchiveException:
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
92 pass
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
93 elif mime_type.startswith('text/'):
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
94 # Should be a plain text file - create a CodeFile for it
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
95 bundle_file = BundleFile(bundle=bundle, name=bundle.file_name,
177
86129d185ddb Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents: 166
diff changeset
96 full_path=bundle.file_name, file_size=os.path.getsize(file),
86129d185ddb Add versioning to bundles
dellsystem <ilostwaldo@gmail.com>
parents: 166
diff changeset
97 version=bundle.latest_version)
151
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
98 bundle_file.save_file_contents(open(file, 'rt'),
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
99 original_filename=bundle.file_name)
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
100
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
101 bundle.done_uploading = True
c7be7def8b57 Bundles! (basic functionality)
dellsystem <ilostwaldo@gmail.com>
parents:
diff changeset
102 bundle.save()