Python on Kyle's Blog

Deep Learning for Coder: Chapter 5

Tue, 16 Sep 2025 00:00:00 +0000

Show/Hide the code

1
2


import fastbook
fastbook.setup_book()

Show/Hide the code

1
2


from fastai.vision.all import *
path = untar_data(URLs.PETS)

Show/Hide the code

1

path.ls()

(#2) [Path('/root/.fastai/data/oxford-iiit-pet/annotations'),Path('/root/.fastai/data/oxford-iiit-pet/images')]

Show/Hide the code

1
2
3
4
5
6
7
8
9


pets = DataBlock(
 blocks=(ImageBlock, CategoryBlock),
 get_items=get_image_files,
 splitter=RandomSplitter(seed=42),
 get_y=using_attr(RegexLabeller(r"(.+)_\d+.jpg$"), "name"),
 item_tfms=Resize(460),
 batch_tfms=aug_transforms(size=224, min_scale=0.75),
)
dls = pets.dataloaders(path/"images")

Show/Hide the code

1

dls.show_batch(nrows=3, ncols=3)

Show/Hide the code

1
2


learn = cnn_learner(dls, resnet34, metrics=error_rate)
learn.fine_tune(2)

Downloading: "https://download.pytorch.org/models/resnet34-b627a593.pth" to /root/.cache/torch/hub/checkpoints/resnet34-b627a593.pth

epoch	train_loss	valid_loss	error_rate	time
0	1.497884	0.302666	0.087280	01:02

epoch	train_loss	valid_loss	error_rate	time
0	0.494630	0.309310	0.099459	01:04
1	0.310687	0.254076	0.084574	01:06

Show/Hide the code

1

x, y = dls.one_batch()

Show/Hide the code

1
2


preds, _ = learn.get_preds(dl = [(x,y)])
torch.argmax(preds[0])

tensor(10)

Show/Hide the code

1
2


acts = torch.randn((6,2))*2
acts

tensor([[ 0.2126, -0.6733],
 [ 0.5868, 1.8977],
 [-1.5544, 0.4964],
 [-0.5226, -0.6044],
 [ 0.4885, -0.1626],
 [-3.0538, -2.0372]])

Show/Hide the code

1

acts.sigmoid().sum(1).unsqueeze(1)

tensor([[0.8907],
 [1.5123],
 [0.7961],
 [0.7256],
 [1.0792],
 [0.1604]])

Show/Hide the code

1

(acts[:,0]-acts[:,1]).sigmoid().unsqueeze(1)

tensor([[0.7080],
 [0.2123],
 [0.1140],
 [0.5204],
 [0.6573],
 [0.2657]])

Show/Hide the code

1
2


def softmax(x): return torch.exp(x) / torch.exp(x).sum(dim=1, keepdim=True)
sm_acts = softmax(acts)

Show/Hide the code

1

sm_acts

tensor([[0.7080, 0.2920],
 [0.2123, 0.7877],
 [0.1140, 0.8860],
 [0.5204, 0.4796],
 [0.6573, 0.3427],
 [0.2657, 0.7343]])

Show/Hide the code

1
2


targ = tensor([0,1,0,0,1,1])
sm_acts[range(6),targ].unsqueeze(1)

tensor([[0.7080],
 [0.7877],
 [0.1140],
 [0.5204],
 [0.3427],
 [0.7343]])

Show/Hide the code

1

-sm_acts[range(6),targ].log().mean()

tensor(0.7981)

Show/Hide the code

1

F.nll_loss(acts.log_softmax(dim=1), targ)

tensor(0.7981)

Show/Hide the code

1
2


loss_func = nn.CrossEntropyLoss(reduction='none')
loss_func(acts, targ)

tensor([0.3452, 0.2387, 2.1718, 0.6531, 1.0708, 0.3088])

Show/Hide the code

1
2


interp = ClassificationInterpretation.from_learner(learn)
interp.plot_confusion_matrix(figsize=(12,12), dpi=60)

Show/Hide the code

1

interp.most_confused(min_val=5)

[('american_pit_bull_terrier', 'staffordshire_bull_terrier', np.int64(12)),
 ('Birman', 'Ragdoll', np.int64(7)),
 ('Egyptian_Mau', 'Bengal', np.int64(5)),
 ('american_pit_bull_terrier', 'american_bulldog', np.int64(5))]

Show/Hide the code

1
2


learn = vision_learner(dls, resnet34, metrics=error_rate)
learn.lr_find()

SuggestedLRs(valley=0.001737800776027143)

Show/Hide the code

1
2


learn = vision_learner(dls, resnet34, metrics=error_rate)
learn.fine_tune(2, base_lr=3e-3)

epoch	train_loss	valid_loss	error_rate	time
0	1.330929	0.305042	0.089986	01:07

epoch	train_loss	valid_loss	error_rate	time
0	0.523440	0.362486	0.104871	01:10
1	0.327184	0.269103	0.080514	01:06

Show/Hide the code

1
2


learn = vision_learner(dls, resnet34, metrics=error_rate)
learn.fit_one_cycle(3, 3e-2)

epoch	train_loss	valid_loss	error_rate	time
0	2.357152	3.695568	0.578484	00:12
1	1.188880	0.541915	0.169824	00:11
2	0.660204	0.358833	0.108931	00:11

Show/Hide the code

1

learn.unfreeze()

Show/Hide the code

1

learn.lr_find()

SuggestedLRs(valley=3.630780702224001e-05)

Show/Hide the code

1

learn.fit_one_cycle(6, lr_max=1e-6)

epoch	train_loss	valid_loss	error_rate	time
0	0.474540	0.356742	0.108931	00:12
1	0.472879	0.353283	0.110284	00:12
2	0.471576	0.347057	0.103518	00:12
3	0.464339	0.345362	0.100812	00:12
4	0.468930	0.342762	0.106225	00:12
5	0.452638	0.343595	0.103518	00:12

Show/Hide the code

1
2
3
4


learn = vision_learner(dls, resnet34, metrics=error_rate)
learn.fit_one_cycle(3, 3e-3)
learn.unfreeze()
learn.fit_one_cycle(12, lr_max=slice(1e-6, 1e-4))

epoch	train_loss	valid_loss	error_rate	time
0	1.127102	0.334892	0.097429	00:11
1	0.518909	0.252394	0.085927	00:11
2	0.327980	0.218656	0.075778	00:11

epoch	train_loss	valid_loss	error_rate	time
0	0.243455	0.213250	0.074425	00:12
1	0.233122	0.212131	0.073748	00:12
2	0.220011	0.202811	0.063599	00:12
3	0.220371	0.196318	0.062923	00:12
4	0.191108	0.200352	0.064953	00:12
5	0.166688	0.201582	0.066982	00:12
6	0.155857	0.196664	0.062923	00:12
7	0.138741	0.190374	0.060217	00:12
8	0.118107	0.192674	0.067659	00:12
9	0.130965	0.191735	0.065629	00:12
10	0.124963	0.191818	0.066982	00:12
11	0.122479	0.191197	0.064276	00:12

Show/Hide the code

1

learn.recorder.plot_loss()

Show/Hide the code

1
2
3


from fastai.callback.fp16 import *
learn = vision_learner(dls, resnet50, metrics=error_rate).to_fp16()
learn.fine_tune(6, freeze_epochs=3)

Downloading: "https://download.pytorch.org/models/resnet50-11ad3fa6.pth" to /root/.cache/torch/hub/checkpoints/resnet50-11ad3fa6.pth

epoch	train_loss	valid_loss	error_rate	time
0	2.191112	0.475555	0.154263	00:14
1	0.829924	0.294044	0.100812	00:13
2	0.550020	0.291130	0.096752	00:13

epoch	train_loss	valid_loss	error_rate	time
0	0.305628	0.236669	0.073748	00:14
1	0.242062	0.278092	0.081867	00:14
2	0.209351	0.231892	0.069689	00:14
3	0.133353	0.220406	0.064276	00:14
4	0.089182	0.202463	0.050744	00:14
5	0.065977	0.192152	0.052774	00:14

Deep Learning for Coder: Chapter 4

Thu, 11 Sep 2025 14:56:41 +1000

Environment Set-up

Show/Hide the code

1
2
3
4
5
6


import fastbook
from fastai.vision.all import *
from fastbook import *

fastbook.setup_book()
matplotlib.rc("image", cmap="Greys")

Load The Two Classes of Data

Show/Hide the code

1
2
3


path = untar_data(URLs.MNIST_SAMPLE)
Path.BASE_PATH = path
path.ls()

(#3) [Path('train'),Path('valid'),Path('labels.csv')]

Show/Hide the code

1
2
3
4
5


threes = [tensor(Image.open(i)) for i in (path / "train" / "3").ls()]
sevens = [tensor(Image.open(i)) for i in (path / "train" / "7").ls()]
threes_valid = [tensor(Image.open(i)) for i in (path / "valid" / "3").ls()]
sevens_valid = [tensor(Image.open(i)) for i in (path / "valid" / "7").ls()]
len(threes), len(sevens)

(6131, 6265)

Show/Hide the code

1
2
3
4
5


threes = torch.stack(threes).float() / 255
sevens = torch.stack(sevens).float() / 255
threes_valid = torch.stack(threes_valid).float() / 255
sevens_valid = torch.stack(sevens_valid).float() / 255
threes.shape, sevens.shape

(torch.Size([6131, 28, 28]), torch.Size([6265, 28, 28]))

Non-Parametric Method

Calculate an “average three”

Show/Hide the code

1
2


mean3 = threes.mean(0)
show_image(mean3)

Calculate an “average seven”

Show/Hide the code

1
2


mean7 = sevens.mean(0)
show_image(mean7)

Define a function that computes distance between two images and a function that judges if an image is three.

Show/Hide the code

1
2
3
4
5
6


def distance(a, b):
 return (a - b).abs().mean((-1, -2))


def is_3(x):
 return distance(x, mean3) < distance(x, mean7)

Compute how many threes are correctly classified into class three and how many seven are correctly classified into seven. (accuracy)

Show/Hide the code

1

is_3(threes_valid).float().mean(), (1 - is_3(sevens_valid).float()).float().mean()

(tensor(0.9168), tensor(0.9854))

Compute overall how many images are correctly classified.

Show/Hide the code

1
2
3
4
5


x = torch.cat([threes_valid, sevens_valid])
y = tensor([1] * len(threes_valid) + [0] * len(sevens_valid))
x.shape, y.shape

# ((distance(x, mean3) < distance(x, mean7)).float() == y).float().mean()

(torch.Size([2038, 28, 28]), torch.Size([2038]))

Linear Function and SGD

Show/Hide the code

1
2
3
4
5


train_x = torch.cat([threes, sevens]).view([-1, 28 * 28])
train_y = tensor([1] * len(threes) + [0] * len(sevens)).unsqueeze(-1)
valid_x = torch.cat([threes_valid, sevens_valid]).view([-1, 28 * 28])
valid_y = tensor([1] * len(threes_valid) + [0] * len(sevens_valid)).unsqueeze(-1)
train_x.shape, train_y.shape, valid_x.shape, valid_y.shape

(torch.Size([12396, 784]),
 torch.Size([12396, 1]),
 torch.Size([2038, 784]),
 torch.Size([2038, 1]))

Show/Hide the code

1
2
3
4
5
6


dset = list(zip(train_x, train_y))
dset_valid = list(zip(valid_x, valid_y))
dset[0][0].shape, dset[0][1].shape

dl = DataLoader(dset, batch_size=256, shuffle=true)
valid_dl = DataLoader(dset_valid, batch_size=256, shuffle=true)

Differentiate unsqueeze, cat and stack:

cat and stack operate on multiple tensors, combining them into a single tensor, whereas unsqueeze operates on a single tensor.
unsqueeze and stack introduce a new dimension to the tensor(s), while cat does not create new dimensions.
stack inserts a new dimension at the specified position, and its length equals the number of tensors being stacked.
cat concatenates tensors along a specified dimension, and the resulting dimension’s length is the sum of the lengths of all tensors along that dimension.
unsqueeze always inserts a new dimension of length 1 at the specified position.

For example:

Show/Hide the code

1
2


a, b = tensor([[1, 2, 3], [4, 5, 6]]), tensor([[7, 8, 9], [1, 2, 1]])
a.shape, b.shape

(torch.Size([2, 3]), torch.Size([2, 3]))

Show/Hide the code

1

a, b

(tensor([[1, 2, 3],
 [4, 5, 6]]),
 tensor([[7, 8, 9],
 [1, 2, 1]]))

cat these two tensors results in a tensor with a shape of either (4, 3) or (2, 6)

Show/Hide the code

1

torch.cat([a, b], 0).shape, torch.cat([a, b], 1).shape

(torch.Size([4, 3]), torch.Size([2, 6]))

stack these two tensors results in a tensor with a shape of either (2, 2, 3) or (2, 3, 2)

Show/Hide the code

1

torch.stack([a, b], 0).shape, torch.stack([a, b], 1).shape, torch.stack([a, b], 2).shape

(torch.Size([2, 2, 3]), torch.Size([2, 2, 3]), torch.Size([2, 3, 2]))

Note: although torch.stack([a, b], 0) and torch.stack([a, b], 1) produce tensors with the same shape, they have different content.

$$ \begin{matrix} [1 & 2 & 3] \\ [4 & 5 & 6] \end{matrix} \quad \begin{matrix} [7 & 8 & 9] \\ [1 & 2 & 1] \end{matrix} $$

torch.stack([a, b], 0) means stack whole matrices:

$$ \left[ \begin{matrix} [1 & 2 & 3] \\ [4 & 5 & 6] \end{matrix} \right] , \left[ \begin{matrix} [7 & 8 & 9] \\ [1 & 2 & 1] \end{matrix} \right] $$

torch.stack([a, b], 1) means stack rows:

$$ \begin{matrix} \textbf{[}[1 & 2 & 3]\quad[7 & 8 & 9]\textbf{]},\\ \\ \textbf{[}[4 & 5 & 6]\quad[1 & 2 & 1]\textbf{]} \end{matrix} $$

torch.stack([a, b], 2) means stack columns.

Show/Hide the code

1

torch.stack([a, b], 0), torch.stack([a, b], 1)

(tensor([[[1, 2, 3],
 [4, 5, 6]],
 
 [[7, 8, 9],
 [1, 2, 1]]]),
 tensor([[[1, 2, 3],
 [7, 8, 9]],
 
 [[4, 5, 6],
 [1, 2, 1]]]))

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33


def sigmoid(logits):
 return 1 / (1 + torch.exp(-logits))


def mnist_loss(logits, targets):
 probs = logits.sigmoid()
 # return ((probs - targets) ** 2).mean().sqrt()
 return torch.where(targets == 1, 1 - probs, probs).mean()


def calc_grad(x, y, f):
 preds = f(x)
 loss = mnist_loss(preds, y)
 loss.backward()


def train_epoch(f, params, lr):
 for xb, yb in dl:
 calc_grad(xb, yb, f)
 for p in params:
 with torch.no_grad():
 p -= lr * p.grad
 p.grad.zero_()


def batch_accuracy(logits, yb):
 probs = logits.sigmoid()
 return ((probs > 0.5) == yb).float().mean()


def validation_epoch(f):
 accs = [batch_accuracy(f(xb), yb) for xb, yb in valid_dl]
 return round(torch.stack(accs).mean().item(), 4)

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16


def init_params(size, std=1.0):
 return (torch.randn(size) * std).requires_grad_()


# column vector
weights = init_params((28 * 28, 1))
bias = init_params(1)


def linear(x):
 return x @ weights + bias


for i in range(20):
 train_epoch(linear, (weights, bias), 1e-1)
 print(validation_epoch(linear))

Optimizer

Show/Hide the code

1
2
3


linear_model = nn.Linear(28 * 28, 1)
w, b = linear_model.parameters()
w.shape, b.shape

(torch.Size([1, 784]), torch.Size([1]))

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26


class BasicOptimizer:
 def __init__(self, params, lr):
 self.params = list(params)
 self.lr = lr

 def step(self, *args, **kwargs):
 for p in self.params:
 with torch.no_grad():
 p -= self.lr * p.grad

 def zero_grad(self, *args, **kwargs):
 for p in self.params:
 p.grad = None


opt = BasicOptimizer(linear_model.parameters(), 1e-2)


def train_epoch(model):
 for xb, yb in dl:
 calc_grad(xb, yb, model)
 opt.step()
 opt.zero_grad()


validation_epoch(linear_model)

0.5231

Show/Hide the code

1
2
3


for i in range(20):
 print(validation_epoch(linear_model))
 train_epoch(linear_model)

Show/Hide the code

1
2
3
4
5


linear_model = nn.Linear(28 * 28, 1)
opt = SGD(linear_model.parameters(), 1.0)
for i in range(20):
 train_epoch(linear_model)
 print(validation_epoch(linear_model))

Learner

Show/Hide the code

1
2
3
4
5
6
7
8
9


dls = DataLoaders(dl, valid_dl)
learn = Learner(
 dls,
 nn.Linear(28 * 28, 1),
 opt_func=SGD,
 loss_func=mnist_loss,
 metrics=batch_accuracy,
)
learn.fit(10, lr=1.0)

epoch	train_loss	valid_loss	batch_accuracy	time
0	0.057635	0.041314	0.971050	00:00
1	0.039915	0.034953	0.974975	00:00
2	0.032067	0.032012	0.974975	00:00
3	0.027766	0.029371	0.977429	00:00
4	0.025469	0.028556	0.977429	00:00
5	0.023680	0.027128	0.978410	00:00
6	0.022530	0.026477	0.978410	00:00
7	0.021128	0.025687	0.978410	00:00
8	0.020596	0.024973	0.978410	00:00
9	0.020155	0.024411	0.979882	00:00

Nonlinearity

Show/Hide the code

1
2
3


simple_net = nn.Sequential(nn.Linear(28 * 28, 30), nn.ReLU(), nn.Linear(30, 1))
learn = Learner(dls, simple_net, mnist_loss, SGD, metrics=batch_accuracy)
learn.fit(40, 0.1)

epoch	train_loss	valid_loss	batch_accuracy	time
0	0.209871	0.085825	0.965653	00:00
1	0.102153	0.052868	0.968106	00:00
2	0.064107	0.043419	0.970069	00:00
3	0.047004	0.038965	0.970559	00:00
4	0.038722	0.036753	0.972522	00:00
5	0.033846	0.034248	0.972522	00:00
6	0.030684	0.032698	0.973994	00:00
7	0.028295	0.031133	0.975957	00:00
8	0.026498	0.030240	0.974975	00:00
9	0.025480	0.029235	0.975957	00:00
10	0.024493	0.028573	0.976938	00:00
11	0.023539	0.028091	0.976448	00:00
12	0.022644	0.027519	0.976938	00:00
13	0.021782	0.026868	0.977429	00:00
14	0.021953	0.026348	0.977920	00:00
15	0.021738	0.025783	0.978901	00:00
16	0.021042	0.025252	0.979392	00:00
17	0.020344	0.024917	0.978901	00:00
18	0.019776	0.024747	0.978901	00:00
19	0.019557	0.024295	0.978901	00:00
20	0.019418	0.024097	0.978901	00:00
21	0.018741	0.023782	0.978901	00:00
22	0.018383	0.023266	0.979392	00:00
23	0.018047	0.023283	0.979882	00:00
24	0.017762	0.023139	0.979882	00:00
25	0.017672	0.022712	0.979882	00:00
26	0.017658	0.022521	0.980864	00:00
27	0.017558	0.022226	0.980864	00:00
28	0.017126	0.022203	0.980864	00:00
29	0.017070	0.021634	0.980864	00:00
30	0.016443	0.021636	0.981354	00:00
31	0.016697	0.021468	0.981354	00:00
32	0.016811	0.021725	0.981845	00:00
33	0.016300	0.021180	0.982336	00:00
34	0.016316	0.021147	0.981845	00:00
35	0.015825	0.020846	0.982336	00:00
36	0.015894	0.020525	0.982336	00:00
37	0.015442	0.020475	0.982336	00:00
38	0.015356	0.020404	0.982336	00:00
39	0.015186	0.020286	0.982336	00:00

Deeper

Show/Hide the code

1
2
3
4
5


dls = ImageDataLoaders.from_folder(path)
learn = vision_learner(
 dls, resnet18, pretrained=False, loss_func=F.cross_entropy, metrics=accuracy
)
learn.fit_one_cycle(1, 0.1)

epoch	train_loss	valid_loss	accuracy	time
0	0.059510	0.017087	0.992640	00:12

Questionnaire

How is a grayscale image represented on a computer? How about a color image?

each pixel is represented by a number from 0 to 255. 0 represents black and 255 represents white. pixels of color image are represented by three colors: red, green and blue (RGB).
How are the files and folders in the MNIST_SAMPLE dataset structured? Why?

tree structure
Explain how the “pixel similarity” approach to classifying digits works.

mean3 and mean7 can be considered as “standard shapes”. if the average distance between an image’s pixels and mean3’s pixels is smaller, it is probably a three and vice versa.
What is a list comprehension? Create one now that selects odd numbers from a list and doubles them.

Create a python list within one line.
Show/Hide the code
1 2

l = [1, 4, 3, 6, 8, 9, 10, 3, 4] [i * 2 for i in l if i % 2 != 0]
```
[2, 6, 18, 6]
```
What is a “rank-3 tensor”?

a tensor that has three dimensions, for example:
Show/Hide the code
1

[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
```
[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
```
What is the difference between tensor rank and shape? How do you get the rank from the shape?

rank is the length of shape, len(shape).
What are RMSE and L1 norm?

RMSE uses square power to avoid cancelling out while L1 norm uses absolute value to do so.
How can you apply a calculation on thousands of numbers at once, many thousands of times faster than a Python loop?

Using np.array or tensor and broadcasting

Create a 3×3 tensor or array containing the numbers from 1 to 9. Double it. Select the bottom-right four numbers.

Show/Hide the code

1
2
3
4
5
6


import numpy as np

a = np.arange(1, 10).reshape(3, 3)
print(a)
a = a * 2
print(a[-2:, -2:])

[[1 2 3]
 [4 5 6]
 [7 8 9]]
[[10 12]
 [16 18]]

What is broadcasting?
Are metrics generally calculated using the training set, or the validation set? Why?

they are generally calculated using the validation set because it refects the generalization ability of a model
What is SGD?

stochastic: stochastically sample batchs from the dataset gradient: update parameters according to its gradient descent: the loss function is convex thus finding minimum descenting
Why does SGD use mini-batches?

Using the whole dataset is computationally expensive and using on data point results in unstable gradient. Using mini-batches is a compromise between them.
What are the seven steps in SGD for machine learning?
How do we initialize the weights in a model?

randomly
What is “loss”?

loss funtions measure the difference between a model’s predictions and the true target values. It provides a quantitative way to evaluate how well a model is performing. lower loss means better performance.
Why can’t we always use a high learning rate?

because it my cause the optimization process to overshoot the minimum. In extreme cases, the loss even increase or diverge instead of decreasing.
What is a “gradient”?
Do you need to know how to calculate gradients yourself?

No, I don’t. Even though I know.
Why can’t we use accuracy as a loss function?

because accuracy is not a continuous function with respect to the model’s parameters, which means when parameters’ value shift a bit, the accuracy won’t change, preventing calculating the gradient.

Draw the sigmoid function. What is special about its shape?

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10


import numpy as np
import matplotlib.pyplot as plt


def sigmoid(x):
 return 1 / (1 + np.exp(-x))


plt.plot(np.arange(-3, 3, 0.1), sigmoid(np.arange(-3, 3, 0.1)))
plt.show()

Its domain is $(-\infty, +\infty)$, and its range is $(0, 1)$. This means it squashes any real value into the interval $(0, 1)$, which makes it suitable for representing probabilities.

What is the difference between a loss function and a metric?

the loss function is used for the model while a metric should be meaingful for humans.
What is the function to calculate new weights using a learning rate?
What does the DataLoader class do?
Write pseudocode showing the basic steps taken in each epoch for SGD.
Create a function that, if passed two arguments [1,2,3,4] and 'abcd', returns [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd')]. What is special about that output data structure?
What does view do in PyTorch?
What are the “bias” parameters in a neural network? Why do we need them?
What does the @ operator do in Python?
What does the backward method do?
Why do we have to zero the gradients?
What information do we have to pass to Learner?
Show Python or pseudocode for the basic steps of a training loop.
What is “ReLU”? Draw a plot of it for values from -2 to +2.
What is an “activation function”?
What’s the difference between F.relu and nn.ReLU?
The universal approximation theorem shows that any function can be approximated as closely as needed using just one nonlinearity. So why do we normally use more?

The Theory of Generalization: From Break Points To The Vc Inequality

Mon, 01 Sep 2025 00:00:00 +0000

The Explanation of Function B

$B(N, k)$ denotes the maximum number of dichotomies on $N$ points, with break point $k$. For example, $B(1000, 4)$ denotes the number of dichotomies that a 2-dimensional perceptron can make on 1000 data points.

Note: A 2-dimensional perceptron has a break point of 4, which means a 2-dimensional perceptron cannot offer all dichotomies on 4 points.

The Recursive Function B

For a dataset with $N$ data points, the total number of possible dichotomies is $2^N$, because each data point has two possible classifications. In this case, the growth rate of the number of dichotomies is an exponential function of $N$. However, due to the existence of a break point, the actual growth rate becomes a polynomial function of $N$, with the degree of the polynomial related to $k$.

Reasoning Process: We divide the $B(N,k)$ dichotomies into two categories based on the following rule: if we ignore the last data point (the $N$-th data point), are the dichotomies of the first $N-1$ data points unique?

Dichotomies that are unique go into Class A.
Dichotomies that are repeated go into Class B.

For example, suppose a dataset has three data points and an algorithm generates four classification methods: 1,0,1, 1,0,0, 0,0,1, 0,1,1. Ignoring the last data point, there are two repeated patterns (1,0) and two unique patterns (0,0 and 0,1).

By definition, Classes A and B are disjoint, and their union is the original set (because each pattern is either unique or repeated).

Let $\alpha$ be the number of dichotomies in Class A.
Since each dichotomy in Class B is repeated exactly twice (the $N$-th data point has two possibilities), let the number of dichotomies in B be $2\beta$.
Thus:

$$ B(N,k) = \alpha + 2\beta $$

Also, $\alpha + \beta$ represents the total number of unique dichotomies for the first $N-1$ data points, so:

$$ \alpha + \beta \le B(N-1,k) $$

The break point of B is $k-1$. This is because if there were $k-1$ columns in B that cover all possible binary patterns, then adding the $N$-th data point would create $k$ columns that cover all possible binary patterns.

For example, if $k=4$, meaning that among all dichotomies, any 4 columns cannot form all binary patterns, and suppose B contains 3 columns:

After adding one more data point, we can find 4 columns that cover all possible binary patterns:

Thus, B cannot have $k-1$ columns covering all binary patterns, meaning the break point is at most $k-1$.

Therefore:

$$ \beta \le B(N-1, k-1) $$

Combining the above, we obtain:

$$ B(N,k) = \alpha + 2\beta \le B(N-1,k) + B(N-1, k-1) $$

The Implementation of Recursive Function B

Show/Hide the code

1
2
3
4
5
6
7
8
9


def B(N, k):

 if k <= 0 or N <= 0:
 return 0
 if k == 1:
 return 1
 if N == 1:
 return 2
 return B(N - 1, k) + B(N - 1, k - 1)

The Table of Function B

Show/Hide the code

1
2
3
4
5
6
7
8
9


print("\t", end="")
for i in range(1, 11):
 print(f"k={i}", end="\t")
print("\n")
for i in range(1, 11):
 print(f"N={i}", end='\t')
 for j in range(1, 11):
 print(B(i, j), end="\t")
 print("\n")

 k=1 k=2 k=3 k=4 k=5 k=6 k=7 k=8 k=9 k=10 

N=1 1 2 2 2 2 2 2 2 2 2 

N=2 1 3 4 4 4 4 4 4 4 4 

N=3 1 4 7 8 8 8 8 8 8 8 

N=4 1 5 11 15 16 16 16 16 16 16 

N=5 1 6 16 26 31 32 32 32 32 32 

N=6 1 7 22 42 57 63 64 64 64 64 

N=7 1 8 29 64 99 120 127 128 128 128 

N=8 1 9 37 93 163 219 247 255 256 256 

N=9 1 10 46 130 256 382 466 502 511 512 

N=10 1 11 56 176 386 638 848 968 1013 1023

A Theorem

$$ B(N,k) \le \sum_{i=0}^{k-1}{\binom{N}{i}} $$

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12


import math
import numpy as np


def binomial(N, i):
 if N < i:
 return 0
 return int(math.factorial(N) / (math.factorial(i) * math.factorial(N - i)))


def B(N, k):
 return np.sum([binomial(N, i) for i in range(0, k)])

Goal:

$$ \sum_{i=0}^{k-1}{\binom{N}{i}}=\sum_{i=0}^{k-1}{\binom{N-1}{i}}+\sum_{i=0}^{k-2}{\binom{N-1}{i}} $$

Steps:

$$ \begin{align} &\sum_{i=0}^{k-1}{\binom{N-1}{i}}+\sum_{i=0}^{k-2}{\binom{N-1}{i}} \nonumber \\ &=1 + \sum_{i=1}^{k-1}{\binom{N-1}{i}}+\sum_{i=1}^{k-2}{\binom{N-1}{i-1}} \nonumber \\ &=1 + \sum_{i=1}^{k-1}{\left[\binom{N-1}{i}+\binom{N-1}{i-1}\right]} \qquad\text{(1)} \nonumber \\ &=1 + \sum_{i=1}^{k-1}{\binom{N}{i}} \qquad\text{(2)} \nonumber \\ &=\sum_{i=0}^{k-1}{\binom{N}{i}} \nonumber \end{align} $$

Note: An intuitive way to understand how (1) becomes to (2) is to consider the essence of the combination formula. The term $\binom{N}{i}$ represents the number of ways to choose $i$ people from a total of $N$. There are two mutually exclusive scenarios for any particular person: either they are chosen, or they are not.

If the person is not chosen, then we need to select all $i$ people from the remaining $N-1$, giving $\binom{N-1}{i}$ combinations. If the person is chosen, we have already selected one, so we only need to choose the remaining $i-1$ people from the other $N-1$, resulting in $\binom{N-1}{i-1}$ combinations. Together, these two cases account for all possible selections.

自动切换 SumatraPDF 主题

Tue, 15 Jul 2025 17:01:11 +0800

SumatraPDF 小巧，快速，对于浏览 PDF 来说再合适不过。唯一的缺点就是功能有些许缺失，例如：虽然有更换主题功能，但是不能跟随 Windows 主题一起变化，并且界面颜色改变时，文档的颜色不改变。于是乎每当在夜晚打开一份 PDF 都仿佛打开了太阳……

微软的应用商店 Microsoft Store 中有一款软件 Auto Dark Mode 可以自动切换 Windows 的暗色模式，并且支持在切换时运行脚本。正巧 SumatraPDF 可以通过配置文件配置，实际上对于一些高级设置（例如字体颜色和背景颜色）只支持使用配置文件设置。因此一个简单的方案就是：编写一个脚本根据参数不同自动修改 SumatraPDF 的配置文件，并且设置 Auto Dark Mode 使其在切换 Windows 主题时自动运行此脚本。

Auto Dark Mode 配置：

Show/Hide the code

1
2
3
4
5
6
7
8
9


Enabled: true
Component:
 Scripts:
 - Name: SumatraPDF
 Command: python
 WorkingDirectory: C:\Users\kyle\AppData\Local\SumatraPDF
 ArgsLight: [toggle_theme.py, light]
 ArgsDark: [toggle_theme.py, dark]
 AllowedSources: [Any]

Python 脚本：

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34


import sys
import os
import re

if len(sys.argv) != 2 or sys.argv[1] not in ['light', 'dark']:
 print("Usage: python toggle_sumatra_theme.py [light|dark]")
 sys.exit(1)

mode = sys.argv[1]

config_path = os.path.join(os.getcwd(), "SumatraPDF-settings.txt")

if not os.path.isfile(config_path):
 print(f"Error: Config file not found: {config_path}")
 sys.exit(1)

if mode == 'light':
 theme_val = 'Light'
 text_color = '#262626'
 background_color = '#f8f3e9'
else:
 theme_val = 'Darker'
 text_color = '#bbbbbb'
 background_color = '#17191a'

with open(config_path, 'r', encoding='utf-8') as f:
 config = f.read()

config = re.sub(r'Theme\s*=\s*\w+', f'Theme = {theme_val}', config)
config = re.sub(r'TextColor\s*=\s*#[0-9a-fA-F]+', f'TextColor = {text_color}', config)
config = re.sub(r'BackgroundColor\s*=\s*#[0-9a-fA-F]+', f'BackgroundColor = {background_color}', config)

with open(config_path, 'w', encoding='utf-8') as f:
 f.write(config)

Download TBBT Transcript From Website

Fri, 13 Jun 2025 12:17:40 +0800

URLs of transcripts

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233


[
 "https://bigbangtrans.wordpress.com/series-1-episode-1-pilot-episode/",
 "https://bigbangtrans.wordpress.com/series-1-episode-2-the-big-bran-hypothesis/",
 "https://bigbangtrans.wordpress.com/series-1-episode-3-the-fuzzy-boots-corollary/",
 "https://bigbangtrans.wordpress.com/series-1-episode-4-the-luminous-fish-effect/",
 "https://bigbangtrans.wordpress.com/series-1-episode-5-the-hamburger-postulate/",
 "https://bigbangtrans.wordpress.com/series-1-episode-6-the-middle-earth-paradigm/",
 "https://bigbangtrans.wordpress.com/series-1-episode-7-the-dumpling-paradox/",
 "https://bigbangtrans.wordpress.com/series-1-episode-8-the-grasshopper-experiment/",
 "https://bigbangtrans.wordpress.com/series-1-episode-9-the-cooper-hofstadter-polarization/",
 "https://bigbangtrans.wordpress.com/series-1-episode-10-the-loobenfeld-decay/",
 "https://bigbangtrans.wordpress.com/series-1-episode-11-the-pancake-batter-anomaly/",
 "https://bigbangtrans.wordpress.com/series-1-episode-12-the-jerusalem-duality/",
 "https://bigbangtrans.wordpress.com/series-1-episode-13-the-bat-jar-conjecture/",
 "https://bigbangtrans.wordpress.com/series-1-episode-14-the-nerdvana-annihilation/",
 "https://bigbangtrans.wordpress.com/series-1-episode-15-the-porkchop-indeterminacy/",
 "https://bigbangtrans.wordpress.com/series-1-episode-16-the-peanut-reaction/",
 "https://bigbangtrans.wordpress.com/series-1-episode-17-the-tangerine-factor/",
 "https://bigbangtrans.wordpress.com/series-2-episode-01-the-bad-fish-paradigm/",
 "https://bigbangtrans.wordpress.com/series-2-episode-02-the-codpiece-topology/",
 "https://bigbangtrans.wordpress.com/series-2-episode-03-the-barbarian-sublimation/",
 "https://bigbangtrans.wordpress.com/series-2-episode-04-the-griffin-equivalency/",
 "https://bigbangtrans.wordpress.com/series-2-episode-5-the-euclid-alternative/",
 "https://bigbangtrans.wordpress.com/series-2-episode-06-the-cooper-nowitzki-theorem-2/",
 "https://bigbangtrans.wordpress.com/series-2-episode-07-the-panty-pinata-polarization/",
 "https://bigbangtrans.wordpress.com/series-1-episode-08-the-lizard-spock-expansion/",
 "https://bigbangtrans.wordpress.com/series-2-episode-09-the-white-asparagus-triangulation/",
 "https://bigbangtrans.wordpress.com/series-2-episode-11-the-vartabedian-conundrum/",
 "https://bigbangtrans.wordpress.com/series-2-episode-11-the-bath-item-gift-hypothesis/",
 "https://bigbangtrans.wordpress.com/series-2-episode-12-the-killer-robot-instability/",
 "https://bigbangtrans.wordpress.com/series-2-episode-13-the-friendship-algorithm/",
 "https://bigbangtrans.wordpress.com/series-2-episode-14-the-financial-permeability-2/",
 "https://bigbangtrans.wordpress.com/series-2-episode-15-the-maternal-capacitance/",
 "https://bigbangtrans.wordpress.com/series-2-episode-16-the-cushion-saturation-3/",
 "https://bigbangtrans.wordpress.com/series-2-episode-17-the-terminator-decoupling/",
 "https://bigbangtrans.wordpress.com/series-2-episode-18-the-work-song-nanocluster/",
 "https://bigbangtrans.wordpress.com/series-2-episode-19-the-dead-hooker-juxtaposition/",
 "https://bigbangtrans.wordpress.com/series-2-episode-20-the-hofstadter-isotope/",
 "https://bigbangtrans.wordpress.com/series-2-episode-21-the-vegas-renormalization/",
 "https://bigbangtrans.wordpress.com/series-2-episode-22-the-classified-materials-turbulence/",
 "https://bigbangtrans.wordpress.com/series-2-episode-23-the-monopolar-expedition/",
 "https://bigbangtrans.wordpress.com/series-3-episode-01-the-electric-can-opener-fluctuation/",
 "https://bigbangtrans.wordpress.com/series-3-episode-02-the-jiminy-conjecture/",
 "https://bigbangtrans.wordpress.com/series-3-episode-03-the-gothowitz-deviation/",
 "https://bigbangtrans.wordpress.com/series-3-episode-04-the-pirate-solution/",
 "https://bigbangtrans.wordpress.com/series-3-episode-05-the-creepy-candy-coating-corollary/",
 "https://bigbangtrans.wordpress.com/series-3-episode-06-the-cornhusker-vortex/",
 "https://bigbangtrans.wordpress.com/series-3-episode-07-the-guitarist-amplification/",
 "https://bigbangtrans.wordpress.com/series-3-episode-08-the-adhesive-duck-deficiency/",
 "https://bigbangtrans.wordpress.com/series-3-episode-09-the-vengeance-formulation/",
 "https://bigbangtrans.wordpress.com/series-3-episode-10-the-gorilla-experiment/",
 "https://bigbangtrans.wordpress.com/series-3-episode-11-the-maternal-congruence/",
 "https://bigbangtrans.wordpress.com/series-3-episode-12-the-psychic-vortex/",
 "https://bigbangtrans.wordpress.com/series-3-episode-13-the-bozeman-reaction/",
 "https://bigbangtrans.wordpress.com/series-3-episode-14-the-einstein-approximation/",
 "https://bigbangtrans.wordpress.com/series-3-episode-15-the-large-hadron-collision/",
 "https://bigbangtrans.wordpress.com/series-3-episode-16-the-excelsior-acquisition/",
 "https://bigbangtrans.wordpress.com/series-3-episode-17-the-precious-fragmentation/",
 "https://bigbangtrans.wordpress.com/series-3-episode-18-the-pants-alternative/",
 "https://bigbangtrans.wordpress.com/series-3-episode-19-the-wheaton-recurrence/",
 "https://bigbangtrans.wordpress.com/series-3-episode-20-the-spaghetti-catalyst/",
 "https://bigbangtrans.wordpress.com/series-3-episode-21-the-plimpton-stimulation/",
 "https://bigbangtrans.wordpress.com/series-3-episode-22-the-staircase-implementation/",
 "https://bigbangtrans.wordpress.com/series-3-episode-23-the-lunar-excitation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-01-the-robotic-manipulation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-02-the-cruciferous-vegetable-amplification/",
 "https://bigbangtrans.wordpress.com/series-4-episode-03-the-zazzy-substitution/",
 "https://bigbangtrans.wordpress.com/series-4-episode-04-the-hot-troll-deviation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-05-the-desperation-emanation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-06-the-irish-pub-formulation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-07-the-apology-insufficiency/",
 "https://bigbangtrans.wordpress.com/series-4-episode-08-the-21-second-excitation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-09-the-boyfriend-complexity/",
 "https://bigbangtrans.wordpress.com/series-4-episode-10-the-alien-parasite-hypothesis/",
 "https://bigbangtrans.wordpress.com/series-4-episode-11-the-justice-league-recombination/",
 "https://bigbangtrans.wordpress.com/series-4-episode-12-the-bus-pants-utilization/",
 "https://bigbangtrans.wordpress.com/series-4-episode-13-the-love-car-displacement/",
 "https://bigbangtrans.wordpress.com/series-4-episode-14-the-thespian-catalyst/",
 "https://bigbangtrans.wordpress.com/series-4-episode-15-the-benefactor-factor/",
 "https://bigbangtrans.wordpress.com/series-4-episode-16-the-cohabitation-formulation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-17-the-toast-derivation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-18-the-prestidigitation-approximation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-19-the-zarnecki-incursion/",
 "https://bigbangtrans.wordpress.com/series-4-episode-20-the-herb-garden-germination/",
 "https://bigbangtrans.wordpress.com/series-4-episode-21-the-agreement-dissection/",
 "https://bigbangtrans.wordpress.com/series-4-episode-22-the-wildebeest-implementation/",
 "https://bigbangtrans.wordpress.com/series-4-episode-23-the-engagement-reaction/",
 "https://bigbangtrans.wordpress.com/series-4-episode-24-the-roommate-transmogrification/",
 "https://bigbangtrans.wordpress.com/series-5-episode-01-the-skank-reflex-analysis/",
 "https://bigbangtrans.wordpress.com/series-5-episode-02-the-infestation-hypothesis/",
 "https://bigbangtrans.wordpress.com/series-5-episode-03-the-pulled-groin-extrapolation/",
 "https://bigbangtrans.wordpress.com/series-5-episode-04-the-wiggly-finger-catalyst/",
 "https://bigbangtrans.wordpress.com/series-5-episode-05-the-russian-rocket-reaction/",
 "https://bigbangtrans.wordpress.com/series-5-episode-06-the-rhinitis-revelation/",
 "https://bigbangtrans.wordpress.com/series-5-episode-07-the-good-guy-fluctuation/",
 "https://bigbangtrans.wordpress.com/series-5-episode-08-the-isolation-permutation/",
 "https://bigbangtrans.wordpress.com/series-5-episode-09-the-ornithophobia-diffusion/",
 "https://bigbangtrans.wordpress.com/series-5-episode-10-the-flaming-spittoon-acquisition/",
 "https://bigbangtrans.wordpress.com/series-5-episode-11-the-speckerman-recurrence/",
 "https://bigbangtrans.wordpress.com/series-5-episode-12-the-shiny-trinket-maneouvre/",
 "https://bigbangtrans.wordpress.com/series-5-episode-13-the-recombination-hypothesis/",
 "https://bigbangtrans.wordpress.com/series-5-episode-14-the-beta-test-initiation/",
 "https://bigbangtrans.wordpress.com/series-5-episode-15-the-friendship-contraction/",
 "https://bigbangtrans.wordpress.com/series-5-episode-16-the-vacation-solution/",
 "https://bigbangtrans.wordpress.com/series-5-episode-17-the-rothman-disintegration/",
 "https://bigbangtrans.wordpress.com/series-5-episode-18-the-werewolf-transformation/",
 "https://bigbangtrans.wordpress.com/series-5-episode-19-the-weekend-vortex/",
 "https://bigbangtrans.wordpress.com/series-5-episode-20-the-transporter-malfunction/",
 "https://bigbangtrans.wordpress.com/series-5-episode-21-the-hawking-excitation/",
 "https://bigbangtrans.wordpress.com/series-5-episode-22-the-stag-convergence/",
 "https://bigbangtrans.wordpress.com/series-5-episode-23-the-launch-acceleration/",
 "https://bigbangtrans.wordpress.com/series-5-episode-24-the-countdown-reflection/",
 "https://bigbangtrans.wordpress.com/series-6-episode-01-the-date-night-variable/",
 "https://bigbangtrans.wordpress.com/series-6-episode-02-the-decoupling-fluctuation/",
 "https://bigbangtrans.wordpress.com/series-6-episode-03-the-higgs-boson-observation/",
 "https://bigbangtrans.wordpress.com/series-6-episode-04-the-re-entry-minimization/",
 "https://bigbangtrans.wordpress.com/series-6-episode-05-the-holographic-excitation/",
 "https://bigbangtrans.wordpress.com/series-6-episode-06-the-extract-obliteration/",
 "https://bigbangtrans.wordpress.com/series-6-episode-07-the-habitation-configuration/",
 "https://bigbangtrans.wordpress.com/series-6-episode-08-the-43-peculiarity/",
 "https://bigbangtrans.wordpress.com/series-6-episode-09-the-parking-spot-escalation/",
 "https://bigbangtrans.wordpress.com/series-6-episode-10-the-fish-guts-displacement/",
 "https://bigbangtrans.wordpress.com/series-6-episode-11-the-santa-simulation/",
 "https://bigbangtrans.wordpress.com/series-6-episode-12-the-egg-salad-equivalency/",
 "https://bigbangtrans.wordpress.com/series-6-episode-13-the-bakersfield-expedition/",
 "https://bigbangtrans.wordpress.com/series-6-episode-14-the-cooperkripke-inversion/",
 "https://bigbangtrans.wordpress.com/series-6-episode-15-the-spoiler-alert-segmentation/",
 "https://bigbangtrans.wordpress.com/series-6-episode-16-the-tangible-affection-proof/",
 "https://bigbangtrans.wordpress.com/series-6-episode-17-the-monster-isolation/",
 "https://bigbangtrans.wordpress.com/series-6-episode-18-the-contractual-obligation-implementation/",
 "https://bigbangtrans.wordpress.com/series-6-episode-19-the-closet-reconfiguration/",
 "https://bigbangtrans.wordpress.com/series-6-episode-20-the-tenure-turbulence/",
 "https://bigbangtrans.wordpress.com/series-6-episode-21-the-closure-alternative/",
 "https://bigbangtrans.wordpress.com/series-6-episode-22-the-proton-resurgence/",
 "https://bigbangtrans.wordpress.com/series-06-episode-23-the-love-spell-potential/",
 "https://bigbangtrans.wordpress.com/series-06-episode-24-the-bon-voyage-reaction/",
 "https://bigbangtrans.wordpress.com/series-7-episode-01-the-hofstadter-insufficiency/",
 "https://bigbangtrans.wordpress.com/series-7-episode-02-the-deception-verification/",
 "https://bigbangtrans.wordpress.com/series-7-episode-03-the-scavenger-vortex/",
 "https://bigbangtrans.wordpress.com/series-7-episode-04-the-raiders-minimization/",
 "https://bigbangtrans.wordpress.com/series-7-episode-05-the-workplace-proximity/",
 "https://bigbangtrans.wordpress.com/series-7-episode-06-the-romance-resonance/",
 "https://bigbangtrans.wordpress.com/series-7-episode-07-the-proton-displacement/",
 "https://bigbangtrans.wordpress.com/series-7-episode-08-the-itchy-brain-simulation/",
 "https://bigbangtrans.wordpress.com/series-7-episode-09-the-thanksgiving-decoupling/",
 "https://bigbangtrans.wordpress.com/series-7-episode-10-the-discovery-dissipation/",
 "https://bigbangtrans.wordpress.com/series-7-episode-11-the-cooper-extraction/",
 "https://bigbangtrans.wordpress.com/series-7-episode-12-the-hesitation-ramification/",
 "https://bigbangtrans.wordpress.com/series-7-episode-13-the-occupation-recalibration/",
 "https://bigbangtrans.wordpress.com/series-7-episode-14-the-convention-conundrum/",
 "https://bigbangtrans.wordpress.com/series-7-episode-15-the-locomotive-manipulation/",
 "https://bigbangtrans.wordpress.com/series-7-episode-16-the-table-polarisation/",
 "https://bigbangtrans.wordpress.com/series-7-episode-17-the-friendship-turbulence/",
 "https://bigbangtrans.wordpress.com/series-7-episode-18-the-mommy-observation/",
 "https://bigbangtrans.wordpress.com/series-7-episode-19-the-indecision-amalgamation/",
 "https://bigbangtrans.wordpress.com/series-7-episode-20-the-relationship-diremption/",
 "https://bigbangtrans.wordpress.com/series-7-episode-21-the-anything-can-happen-recurrence/",
 "https://bigbangtrans.wordpress.com/series-7-episode-22-the-proton-transmogrification/",
 "https://bigbangtrans.wordpress.com/series-7-episode-23-the-gorilla-dissolution/",
 "https://bigbangtrans.wordpress.com/series-7-episode-24-the-status-quo-combustion/",
 "https://bigbangtrans.wordpress.com/series-8-episode-01-the-locomotion-interruption/",
 "https://bigbangtrans.wordpress.com/series-8-episode-02-the-junior-professor-solution/",
 "https://bigbangtrans.wordpress.com/series-8-episode-03-the-first-pitch-insufficiency/",
 "https://bigbangtrans.wordpress.com/series-8-episode-04-the-hook-up-reverbration/",
 "https://bigbangtrans.wordpress.com/series-7-episode-05-the-focus-attenuation/",
 "https://bigbangtrans.wordpress.com/series-8-episode-06-the-expedition-approximation/",
 "https://bigbangtrans.wordpress.com/series-8-episode-07-the-misinterpretation-agitation/",
 "https://bigbangtrans.wordpress.com/series-8-episode-08-the-prom-equivalency/",
 "https://bigbangtrans.wordpress.com/series-8-episode-09-the-septum-deviation/",
 "https://bigbangtrans.wordpress.com/series-8-episode-10-the-champagne-reflection/",
 "https://bigbangtrans.wordpress.com/series-8-episode-11-the-clean-room-infiltration/",
 "https://bigbangtrans.wordpress.com/series-8-episode-12-the-space-probe-disintegration/",
 "https://bigbangtrans.wordpress.com/series-8-episode-13-the-anxiety-optimisation/",
 "https://bigbangtrans.wordpress.com/series-8-episode-14-the-troll-manifestation/",
 "https://bigbangtrans.wordpress.com/series-8-episode-15-the-comic-book-store-regeneration/",
 "https://bigbangtrans.wordpress.com/series-8-episode-16-the-intimacy-acceleration/",
 "https://bigbangtrans.wordpress.com/series-8-episode-17-the-colonization-application/",
 "https://bigbangtrans.wordpress.com/series-8-episode-18-the-leftover-thermalization/",
 "https://bigbangtrans.wordpress.com/series-8-episode-19-the-skywalker-incursion/",
 "https://bigbangtrans.wordpress.com/series-8-episode-20-the-fortification-implementation/",
 "https://bigbangtrans.wordpress.com/series-8-episode-21-the-communication-deterioration/",
 "https://bigbangtrans.wordpress.com/series-8-episode-22-the-graduation-transmission/",
 "https://bigbangtrans.wordpress.com/series-8-episode-23-the-maternal-combustion/",
 "https://bigbangtrans.wordpress.com/series-8-episode-24-the-commitment-determination/",
 "https://bigbangtrans.wordpress.com/series-9-episode-01-the-matrimonial-momentum/",
 "https://bigbangtrans.wordpress.com/series-9-episode-02-the-separation-oscillation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-03-the-batchelor-party-corrosion/",
 "https://bigbangtrans.wordpress.com/series-9-episode-04-the-2003-approximation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-05-the-perspiration-implementation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-06-the-helium-insufficiency/",
 "https://bigbangtrans.wordpress.com/series-9-episode-07-the-spock-resonance/",
 "https://bigbangtrans.wordpress.com/series-9-episode-08-the-mystery-date-observation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-09-the-platonic-permutation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-10-the-earworm-reverberation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-11-the-opening-night-excitation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-12-the-sales-call-sublimation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-13-the-empathy-optimisation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-14-the-meemaw-materialisation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-15-the-valentino-submergence/",
 "https://bigbangtrans.wordpress.com/series-9-episode-16-the-positive-negative-reaction/",
 "https://bigbangtrans.wordpress.com/series-9-episode-17-the-celebration-experimentation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-18-the-application-deterioration/",
 "https://bigbangtrans.wordpress.com/series-9-episode-19-the-solder-excursion-diversion/",
 "https://bigbangtrans.wordpress.com/series-9-episode-20-the-big-bear-precipitation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-21-the-viewing-party-combustion/",
 "https://bigbangtrans.wordpress.com/series-9-episode-22-the-fermentation-bifurcation/",
 "https://bigbangtrans.wordpress.com/series-9-episode-23-the-line-substitution-solution/",
 "https://bigbangtrans.wordpress.com/series-9-episode-24-the-convergence-convergence/",
 "https://bigbangtrans.wordpress.com/series-10-episode-01-the-conjugal-conjecture/",
 "https://bigbangtrans.wordpress.com/series-10-episode-02-the-military-miniturization/",
 "https://bigbangtrans.wordpress.com/series-10-episode-03-the-dependence-transcendence/",
 "https://bigbangtrans.wordpress.com/series-10-episode-04-the-cohabitation-experimentation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-05-the-hot-tub-contamination/",
 "https://bigbangtrans.wordpress.com/series-10-episode-06-the-foetal-kick-catalyst/",
 "https://bigbangtrans.wordpress.com/series-10-episode-07-the-veracity-elasticity/",
 "https://bigbangtrans.wordpress.com/series-10-episode-08-the-brain-bowl-incubation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-09-the-geology-elevation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-10-the-property-division-collision/",
 "https://bigbangtrans.wordpress.com/series-10-episode-11-the-birthday-synchronicity/",
 "https://bigbangtrans.wordpress.com/series-10-episode-12-the-holiday-summation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-13-the-romance-recalibration/",
 "https://bigbangtrans.wordpress.com/series-10-episode-14-the-emotion-detection-automation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-15-the-locomotion-reverberation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-16-the-allowance-evaporation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-17-the-comic-con-conundrum/",
 "https://bigbangtrans.wordpress.com/series-10-episode-18-the-escape-hatch-identification/",
 "https://bigbangtrans.wordpress.com/series-10-episode-19-the-collaboration-fluctuation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-20-the-recollection-dissipation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-21-the-separation-agitation/",
 "https://bigbangtrans.wordpress.com/series-10-episode-22-the-cognition-regeneration/",
 "https://bigbangtrans.wordpress.com/series-10-episode-23-the-gyroscopic-collapse/",
 "https://bigbangtrans.wordpress.com/series-10-episode-24-the-long-distance-dissonance/"
]

Script

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32


import requests
from bs4 import BeautifulSoup
import os

url_list = []

def download_script(url):
 response = requests.get(url)
 response.encoding = "utf-8"
 soup = BeautifulSoup(response.text, "html.parser")

 script_text = ""
 title = soup.find("h2", class_="title").get_text()
 script_text = script_text + "## " + title + "\n\n"
 div = soup.find("div", class_="entrytext")
 p_tags = div.find_all("p")

 for p in p_tags:
	 text = p.get_text()
	 print(text)
	 try:
		 before, after = text.split(":", 1)
		 script_text = script_text + "**" + before + "**" + ": " + after + "\n\n"
	 except:
		 script_text = script_text + text + "\n\n"

 return script_text


with open("TBBT.md", "a", encoding="utf-8") as f:
 for url in url_list:
	 print(download_script(url), file=f)

Convert Jupyter Notebook to PDF With Reporting Format

Fri, 13 Jun 2025 00:00:00 +0000

Install nbconvert via pip

Show/Hide the code

1

pip install nbconvert

Now, the command

Show/Hide the code

1

jupyter nbconvert --to pdf --template-file report filename.ipynb

should work well, producing a document with a table of contents and properly formatted code blocks. However, an unnecessary counter appears before each code block, wasting space and serving no purpose. To remove it, a custom template is required.

The template consists of two files: assignment.tex.j2 and style_assignment.tex.j2, which are modified duplicates of existing files. Template files are located in share\jupyter\nbconvert\templates\latex. Duplicate report.tex.j2 and modify the second line to

Show/Hide the code

1

((* set cell_style = 'style_assignment.tex.j2' *))

The file style_assignment.tex.j2 is a duplicate of style_jupyter.tex.j2 with the line

Show/Hide the code

1

{\ttfamily\llap{{\color{#2}[#3]:\hspace{3pt}#4}}\vspace{-\baselineskip}}

replaced by

Show/Hide the code

1

{\ttfamily\llap{{\color{#2}\hspace{3pt}#4}}}

Moreover, in order to support Chinese, the code below should be added

Show/Hide the code

1
2


\usepackage{xeCJK}
\setCJKmainfont{SimSun}

Now that custom files are used, the command is

Show/Hide the code

1

jupyter nbconvert --to pdf --template-file assignment filename.ipynb

or dictate author names and the document title as converting:

Show/Hide the code

1

jupyter nbconvert --to pdf --template-file assignment ./Journal.ipynb --LatexPreprocessor.author_names="Name" --LatexPreprocessor.title="Title"

The complete file is shown as follows:

assignment.tex.j2

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25


% Default to the notebook output style
((* if not cell_style is defined *))
 ((* set cell_style = 'style_assignment.tex.j2' *))
((* endif *))

% Inherit from the specified cell style.
((* extends cell_style *))


%===============================================================================
% Latex Book
%===============================================================================

((* block predoc *))
 ((( super() )))
 ((* block tableofcontents *))\tableofcontents((* endblock tableofcontents *))
((* endblock predoc *))

((* block docclass *))
\documentclass{report}
((* endblock docclass *))

((* block markdowncell scoped *))
((( cell.source | citation2latex | strip_files_prefix | convert_pandoc('markdown+tex_math_double_backslash', 'json',extra_args=[]) | resolve_references | convert_pandoc('json','latex', extra_args=["--top-level-division=chapter"]) )))
((* endblock markdowncell *))

style_assignment.tex.j2

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180


((=- IPython input/output style -=))
((*- extends 'base.tex.j2' -*))

((*- block packages -*))
	\usepackage{xeCJK}
	\setCJKmainfont{SimSun}
 \usepackage[breakable]{tcolorbox}
 \usepackage{parskip} % Stop auto-indenting (to mimic markdown behaviour)
 ((( super() )))
((*- endblock packages -*))

((*- block definitions -*))
 ((( super() )))
% Pygments definitions
 (((- resources.latex.pygments_definitions )))

 % For linebreaks inside Verbatim environment from package fancyvrb.
 \makeatletter
 \newbox\Wrappedcontinuationbox
 \newbox\Wrappedvisiblespacebox
 \newcommand*\Wrappedvisiblespace {\textcolor{red}{\textvisiblespace}}
 \newcommand*\Wrappedcontinuationsymbol {\textcolor{red}{\llap{\tiny$\m@th\hookrightarrow$}}}
 \newcommand*\Wrappedcontinuationindent {3ex }
 \newcommand*\Wrappedafterbreak {\kern\Wrappedcontinuationindent\copy\Wrappedcontinuationbox}
 % Take advantage of the already applied Pygments mark-up to insert
 % potential linebreaks for TeX processing.
 % {, <, #, %, $, ' and ": go to next line.
 % _, }, ^, &, >, - and ~: stay at end of broken line.
 % Use of \textquotesingle for straight quote.
 \newcommand*\Wrappedbreaksatspecials {%
 \def\PYGZus{\discretionary{\char`\_}{\Wrappedafterbreak}{\char`\_}}%
 \def\PYGZob{\discretionary{}{\Wrappedafterbreak\char`\{}{\char`\{}}%
 \def\PYGZcb{\discretionary{\char`\}}{\Wrappedafterbreak}{\char`\}}}%
 \def\PYGZca{\discretionary{\char`\^}{\Wrappedafterbreak}{\char`\^}}%
 \def\PYGZam{\discretionary{\char`\&}{\Wrappedafterbreak}{\char`\&}}%
 \def\PYGZlt{\discretionary{}{\Wrappedafterbreak\char`\<}{\char`\<}}%
 \def\PYGZgt{\discretionary{\char`\>}{\Wrappedafterbreak}{\char`\>}}%
 \def\PYGZsh{\discretionary{}{\Wrappedafterbreak\char`\#}{\char`\#}}%
 \def\PYGZpc{\discretionary{}{\Wrappedafterbreak\char`\%}{\char`\%}}%
 \def\PYGZdl{\discretionary{}{\Wrappedafterbreak\char`\$}{\char`\$}}%
 \def\PYGZhy{\discretionary{\char`\-}{\Wrappedafterbreak}{\char`\-}}%
 \def\PYGZsq{\discretionary{}{\Wrappedafterbreak\textquotesingle}{\textquotesingle}}%
 \def\PYGZdq{\discretionary{}{\Wrappedafterbreak\char`\"}{\char`\"}}%
 \def\PYGZti{\discretionary{\char`\~}{\Wrappedafterbreak}{\char`\~}}%
 }
 % Some characters . , ; ? ! / are not pygmentized.
 % This macro makes them "active" and they will insert potential linebreaks
 \newcommand*\Wrappedbreaksatpunct {%
 \lccode`\~`\.\lowercase{\def~}{\discretionary{\hbox{\char`\.}}{\Wrappedafterbreak}{\hbox{\char`\.}}}%
 \lccode`\~`\,\lowercase{\def~}{\discretionary{\hbox{\char`\,}}{\Wrappedafterbreak}{\hbox{\char`\,}}}%
 \lccode`\~`\;\lowercase{\def~}{\discretionary{\hbox{\char`\;}}{\Wrappedafterbreak}{\hbox{\char`\;}}}%
 \lccode`\~`\:\lowercase{\def~}{\discretionary{\hbox{\char`\:}}{\Wrappedafterbreak}{\hbox{\char`\:}}}%
 \lccode`\~`\?\lowercase{\def~}{\discretionary{\hbox{\char`\?}}{\Wrappedafterbreak}{\hbox{\char`\?}}}%
 \lccode`\~`\!\lowercase{\def~}{\discretionary{\hbox{\char`\!}}{\Wrappedafterbreak}{\hbox{\char`\!}}}%
 \lccode`\~`\/\lowercase{\def~}{\discretionary{\hbox{\char`\/}}{\Wrappedafterbreak}{\hbox{\char`\/}}}%
 \catcode`\.\active
 \catcode`\,\active
 \catcode`\;\active
 \catcode`\:\active
 \catcode`\?\active
 \catcode`\!\active
 \catcode`\/\active
 \lccode`\~`\~
 }
 \makeatother

 \let\OriginalVerbatim=\Verbatim
 \makeatletter
 \renewcommand{\Verbatim}[1][1]{%
 %\parskip\z@skip
 \sbox\Wrappedcontinuationbox {\Wrappedcontinuationsymbol}%
 \sbox\Wrappedvisiblespacebox {\FV@SetupFont\Wrappedvisiblespace}%
 \def\FancyVerbFormatLine ##1{\hsize\linewidth
 \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@
 \doublehyphendemerits\z@\finalhyphendemerits\z@
 \strut ##1\strut}%
 }%
 % If the linebreak is at a space, the latter will be displayed as visible
 % space at end of first line, and a continuation symbol starts next line.
 % Stretch/shrink are however usually zero for typewriter font.
 \def\FV@Space {%
 \nobreak\hskip\z@ plus\fontdimen3\font minus\fontdimen4\font
 \discretionary{\copy\Wrappedvisiblespacebox}{\Wrappedafterbreak}
 {\kern\fontdimen2\font}%
 }%

 % Allow breaks at special characters using \PYG... macros.
 \Wrappedbreaksatspecials
 % Breaks at punctuation characters . , ; ? ! and / need catcode=\active
 \OriginalVerbatim[#1,codes*=\Wrappedbreaksatpunct]%
 }
 \makeatother

 % Exact colors from NB
 ((*- block style_colors *))
 \definecolor{incolor}{HTML}{303F9F}
 \definecolor{outcolor}{HTML}{D84315}
 \definecolor{cellborder}{HTML}{CFCFCF}
 \definecolor{cellbackground}{HTML}{F7F7F7}
 ((*- endblock style_colors *))

 % prompt
 \makeatletter
 \newcommand{\boxspacing}{\kern\kvtcb@left@rule\kern\kvtcb@boxsep}
 \makeatother
 ((*- block style_prompt *))
 \newcommand{\prompt}[4]{
 {\ttfamily\llap{{\color{#2}\hspace{3pt}#4}}}
 }
 ((* endblock style_prompt *))

((*- endblock definitions -*))

%===============================================================================
% Input
%===============================================================================

((* block input scoped *))
 ((( draw_cell(cell.source | highlight_code(strip_verbatim=True), cell, 'In', 'incolor', '\\boxspacing') )))
((* endblock input *))


%===============================================================================
% Output
%===============================================================================

((*- if charlim is not defined -*))
 ((* set charlim = 80 *))
((*- endif -*))

((* block execute_result scoped *))
 ((*- for type in output.data | filter_data_type -*))
 ((*- if type in ['text/plain']*))
 ((( draw_cell(output.data['text/plain'] | wrap_text(charlim) | escape_latex | ansi2latex, cell, 'Out', 'outcolor', '\\boxspacing') )))
 ((* else -*))
 ((( " " )))
 ((( draw_prompt(cell, 'Out', 'outcolor','') )))((( super() )))
 ((*- endif -*))
 ((*- endfor -*))
((* endblock execute_result *))

((* block stream *))
 \begin{Verbatim}[commandchars=\\\{\}]
((( output.text | wrap_text(charlim) | escape_latex | strip_trailing_newline | ansi2latex )))
 \end{Verbatim}
((* endblock stream *))

%==============================================================================
% Support Macros
%==============================================================================

% Name: draw_cell
% Purpose: Renders an output/input prompt
((*- if draw_cell is not defined -*)) % Required to allow overriding.
((* macro draw_cell(text, cell, prompt, prompt_color, extra_space) -*))
((*- if prompt == 'In' -*))
((*- set style = "breakable, size=fbox, boxrule=1pt, pad at break*=1mm,colback=cellbackground, colframe=cellborder"-*))
((*- else -*))((*- set style = "breakable, size=fbox, boxrule=.5pt, pad at break*=1mm, opacityfill=0"-*))((*- endif -*))

\begin{tcolorbox}[((( style )))]
(((- draw_prompt(cell, prompt, prompt_color, extra_space) )))
\begin{Verbatim}[commandchars=\\\{\}]
((( text )))
\end{Verbatim}
\end{tcolorbox}
((*- endmacro *))
((*- endif -*))

% Name: draw_prompt
% Purpose: Renders an output/input prompt
((* macro draw_prompt(cell, prompt, prompt_color, extra_space) -*))
 ((*- if cell.execution_count is defined -*))
 ((*- set execution_count = "" ~ (cell.execution_count | replace(None, " ")) -*))
 ((*- else -*))((*- set execution_count = " " -*))((*- endif *))

 ((*- if (resources.global_content_filter.include_output_prompt and prompt == 'Out')
 or (resources.global_content_filter.include_input_prompt and prompt == 'In' ) *))
\prompt{(((prompt)))}{(((prompt_color)))}{(((execution_count)))}{(((extra_space)))}
 ((*- endif -*))
((*- endmacro *))

Furthermore, if you want to hide code:

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181


((=- IPython input/output style -=))
((*- extends 'base.tex.j2' -*))

((*- block packages -*))
	\usepackage{xeCJK}
	\setCJKmainfont{SimSun}
 \usepackage[breakable]{tcolorbox}
 \usepackage{parskip} % Stop auto-indenting (to mimic markdown behaviour)
 ((( super() )))
((*- endblock packages -*))

((*- block definitions -*))
 ((( super() )))
% Pygments definitions
 (((- resources.latex.pygments_definitions )))

 % For linebreaks inside Verbatim environment from package fancyvrb.
 \makeatletter
 \newbox\Wrappedcontinuationbox
 \newbox\Wrappedvisiblespacebox
 \newcommand*\Wrappedvisiblespace {\textcolor{red}{\textvisiblespace}}
 \newcommand*\Wrappedcontinuationsymbol {\textcolor{red}{\llap{\tiny$\m@th\hookrightarrow$}}}
 \newcommand*\Wrappedcontinuationindent {3ex }
 \newcommand*\Wrappedafterbreak {\kern\Wrappedcontinuationindent\copy\Wrappedcontinuationbox}
 % Take advantage of the already applied Pygments mark-up to insert
 % potential linebreaks for TeX processing.
 % {, <, #, %, $, ' and ": go to next line.
 % _, }, ^, &, >, - and ~: stay at end of broken line.
 % Use of \textquotesingle for straight quote.
 \newcommand*\Wrappedbreaksatspecials {%
 \def\PYGZus{\discretionary{\char`\_}{\Wrappedafterbreak}{\char`\_}}%
 \def\PYGZob{\discretionary{}{\Wrappedafterbreak\char`\{}{\char`\{}}%
 \def\PYGZcb{\discretionary{\char`\}}{\Wrappedafterbreak}{\char`\}}}%
 \def\PYGZca{\discretionary{\char`\^}{\Wrappedafterbreak}{\char`\^}}%
 \def\PYGZam{\discretionary{\char`\&}{\Wrappedafterbreak}{\char`\&}}%
 \def\PYGZlt{\discretionary{}{\Wrappedafterbreak\char`\<}{\char`\<}}%
 \def\PYGZgt{\discretionary{\char`\>}{\Wrappedafterbreak}{\char`\>}}%
 \def\PYGZsh{\discretionary{}{\Wrappedafterbreak\char`\#}{\char`\#}}%
 \def\PYGZpc{\discretionary{}{\Wrappedafterbreak\char`\%}{\char`\%}}%
 \def\PYGZdl{\discretionary{}{\Wrappedafterbreak\char`\$}{\char`\$}}%
 \def\PYGZhy{\discretionary{\char`\-}{\Wrappedafterbreak}{\char`\-}}%
 \def\PYGZsq{\discretionary{}{\Wrappedafterbreak\textquotesingle}{\textquotesingle}}%
 \def\PYGZdq{\discretionary{}{\Wrappedafterbreak\char`\"}{\char`\"}}%
 \def\PYGZti{\discretionary{\char`\~}{\Wrappedafterbreak}{\char`\~}}%
 }
 % Some characters . , ; ? ! / are not pygmentized.
 % This macro makes them "active" and they will insert potential linebreaks
 \newcommand*\Wrappedbreaksatpunct {%
 \lccode`\~`\.\lowercase{\def~}{\discretionary{\hbox{\char`\.}}{\Wrappedafterbreak}{\hbox{\char`\.}}}%
 \lccode`\~`\,\lowercase{\def~}{\discretionary{\hbox{\char`\,}}{\Wrappedafterbreak}{\hbox{\char`\,}}}%
 \lccode`\~`\;\lowercase{\def~}{\discretionary{\hbox{\char`\;}}{\Wrappedafterbreak}{\hbox{\char`\;}}}%
 \lccode`\~`\:\lowercase{\def~}{\discretionary{\hbox{\char`\:}}{\Wrappedafterbreak}{\hbox{\char`\:}}}%
 \lccode`\~`\?\lowercase{\def~}{\discretionary{\hbox{\char`\?}}{\Wrappedafterbreak}{\hbox{\char`\?}}}%
 \lccode`\~`\!\lowercase{\def~}{\discretionary{\hbox{\char`\!}}{\Wrappedafterbreak}{\hbox{\char`\!}}}%
 \lccode`\~`\/\lowercase{\def~}{\discretionary{\hbox{\char`\/}}{\Wrappedafterbreak}{\hbox{\char`\/}}}%
 \catcode`\.\active
 \catcode`\,\active
 \catcode`\;\active
 \catcode`\:\active
 \catcode`\?\active
 \catcode`\!\active
 \catcode`\/\active
 \lccode`\~`\~
 }
 \makeatother

 \let\OriginalVerbatim=\Verbatim
 \makeatletter
 \renewcommand{\Verbatim}[1][1]{%
 %\parskip\z@skip
 \sbox\Wrappedcontinuationbox {\Wrappedcontinuationsymbol}%
 \sbox\Wrappedvisiblespacebox {\FV@SetupFont\Wrappedvisiblespace}%
 \def\FancyVerbFormatLine ##1{\hsize\linewidth
 \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@
 \doublehyphendemerits\z@\finalhyphendemerits\z@
 \strut ##1\strut}%
 }%
 % If the linebreak is at a space, the latter will be displayed as visible
 % space at end of first line, and a continuation symbol starts next line.
 % Stretch/shrink are however usually zero for typewriter font.
 \def\FV@Space {%
 \nobreak\hskip\z@ plus\fontdimen3\font minus\fontdimen4\font
 \discretionary{\copy\Wrappedvisiblespacebox}{\Wrappedafterbreak}
 {\kern\fontdimen2\font}%
 }%

 % Allow breaks at special characters using \PYG... macros.
 \Wrappedbreaksatspecials
 % Breaks at punctuation characters . , ; ? ! and / need catcode=\active
 \OriginalVerbatim[#1,codes*=\Wrappedbreaksatpunct]%
 }
 \makeatother

 % Exact colors from NB
 ((*- block style_colors *))
 \definecolor{incolor}{HTML}{303F9F}
 \definecolor{outcolor}{HTML}{D84315}
 \definecolor{cellborder}{HTML}{CFCFCF}
 \definecolor{cellbackground}{HTML}{F7F7F7}
 ((*- endblock style_colors *))

 % prompt
 \makeatletter
 \newcommand{\boxspacing}{\kern\kvtcb@left@rule\kern\kvtcb@boxsep}
 \makeatother
 ((*- block style_prompt *))
 \newcommand{\prompt}[4]{
 {\ttfamily\llap{{\color{#2}\hspace{3pt}#4}}}
 }
 ((* endblock style_prompt *))

((*- endblock definitions -*))

%===============================================================================
% Input
%===============================================================================

((* block input scoped *))
 % input hidden
((* endblock input *))



%===============================================================================
% Output
%===============================================================================

((*- if charlim is not defined -*))
 ((* set charlim = 80 *))
((*- endif -*))

((* block execute_result scoped *))
 ((*- for type in output.data | filter_data_type -*))
 ((*- if type in ['text/plain']*))
 ((( draw_cell(output.data['text/plain'] | wrap_text(charlim) | escape_latex | ansi2latex, cell, 'Out', 'outcolor', '\\boxspacing') )))
 ((* else -*))
 ((( " " )))
 ((( draw_prompt(cell, 'Out', 'outcolor','') )))((( super() )))
 ((*- endif -*))
 ((*- endfor -*))
((* endblock execute_result *))

((* block stream *))
 \begin{Verbatim}[commandchars=\\\{\}]
((( output.text | wrap_text(charlim) | escape_latex | strip_trailing_newline | ansi2latex )))
 \end{Verbatim}
((* endblock stream *))

%==============================================================================
% Support Macros
%==============================================================================

% Name: draw_cell
% Purpose: Renders an output/input prompt
((*- if draw_cell is not defined -*)) % Required to allow overriding.
((* macro draw_cell(text, cell, prompt, prompt_color, extra_space) -*))
((*- if prompt == 'In' -*))
((*- set style = "breakable, size=fbox, boxrule=1pt, pad at break*=1mm,colback=cellbackground, colframe=cellborder"-*))
((*- else -*))((*- set style = "breakable, size=fbox, boxrule=.5pt, pad at break*=1mm, opacityfill=0"-*))((*- endif -*))

\begin{tcolorbox}[((( style )))]
(((- draw_prompt(cell, prompt, prompt_color, extra_space) )))
\begin{Verbatim}[commandchars=\\\{\}]
((( text )))
\end{Verbatim}
\end{tcolorbox}
((*- endmacro *))
((*- endif -*))

% Name: draw_prompt
% Purpose: Renders an output/input prompt
((* macro draw_prompt(cell, prompt, prompt_color, extra_space) -*))
 ((*- if cell.execution_count is defined -*))
 ((*- set execution_count = "" ~ (cell.execution_count | replace(None, " ")) -*))
 ((*- else -*))((*- set execution_count = " " -*))((*- endif *))

 ((*- if (resources.global_content_filter.include_output_prompt and prompt == 'Out')
 or (resources.global_content_filter.include_input_prompt and prompt == 'In' ) *))
\prompt{(((prompt)))}{(((prompt_color)))}{(((execution_count)))}{(((extra_space)))}
 ((*- endif -*))
((*- endmacro *))

Merge Anki Audios

Fri, 13 Jun 2025 00:00:00 +0000

exporting all selected notes with media references

extract media paths from the exported file
write a python script merging all selected audios into one

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79


import os
import subprocess
import pandas as pd
from concurrent.futures import ThreadPoolExecutor, as_completed
from tqdm import tqdm
import random

AUDIO_DIRECTORY = r"C:\Users\kyle\AppData\Roaming\Anki2\Eng\collection.media"
SILENT = r"C:\Users\momo\Home\Projects\Anki\silent.mp3"

list_file_path = "../notes.txt"
df = pd.read_csv(
 list_file_path,
 sep="\t",
 header=None,
 names=[
 "word", "lemma", "context", "sentence", "examples", "symbol", "voc", "def",
 "word_sound", "sentence_sound"
 ],
)

word_audio_names = df["word_sound"].apply(lambda x: x.strip()[7:-1]).tolist()
df["sentence_sound"] = df["sentence_sound"].fillna("")
sentence_audio_names = df["sentence_sound"].apply(lambda x: x.strip()[7:-1]).tolist()

audio_paths = [os.path.join(AUDIO_DIRECTORY, item) for item in word_audio_names] + [
 os.path.join(AUDIO_DIRECTORY, item) for item in sentence_audio_names if item != ""
]


def normalize_audio(input_file, output_file):
 try:
 command = [
 "ffmpeg", "-loglevel", "quiet", "-y", "-i", input_file, "-ar", "24000",
 "-ab", "96k", output_file
 ]
 subprocess.run(command, check=True)
 return output_file
 except subprocess.CalledProcessError as e:
 return None


def normalize_audio_multithreaded(file_list, output_dir, max_workers=16):
 if not os.path.exists(output_dir):
 os.makedirs(output_dir)
 print("start normalizing audios")
 with ThreadPoolExecutor(max_workers=max_workers) as executor:
 futures = [
 executor.submit(normalize_audio, input_file,
 os.path.join(output_dir, os.path.basename(input_file)))
 for input_file in file_list
 ]
 for _ in tqdm(as_completed(futures), total=len(file_list)):
 pass
 else:
 print("files has prepared well")


normalize_audio_multithreaded(audio_paths, "temp")

audio_paths = [(w, s) for w, s in zip(word_audio_names, sentence_audio_names)]
random.shuffle(audio_paths)

with open("file_list.txt", "w") as f:
 for w, s in audio_paths:
 f.write(f"file '{os.path.join('temp', w)}'\n")
 f.write(f"file silent.mp3\n")
 f.write(f"file '{os.path.join('temp', w)}'\n")
 f.write(f"file silent.mp3\n")
 if s != "":
 f.write(f"file '{os.path.join('temp', s)}'\n")
 f.write(f"file silent.mp3\n")
 f.write(f"file silent.mp3\n")

command = [
 "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", "file_list.txt", "-c", "copy",
 "output.mp3"
]
subprocess.run(command, check=True)

Prune a Dictionary File(.mdx)

Fri, 13 Jun 2025 00:00:00 +0000

Introduction

Dictionaries such as “Oxford” and “Longman” are used as the sources for the definitions in my flashcards. However, they sometimes contain too much unnecessary information, therefore making it unwieldy and grandiose, leading to a low loading speed and distractions.

Since an MDX file utilizes the HTML format to save entries, which closely resembles the tree structure, I use the term “prune” to describe the course of removing branches, i.e. html nodes that contain superfluous elements.

Each group of three lines in a mdx file constitutes an entry of a word where the first and the third lines are delimiters while the middle line is html content. Since typically MDX files are quite large, a division step is required for parallel processing. Thus, the breakdown procedures are as follows:

split a gigantic mdx file into several small files. Note: the number of lines in each file must be a multiple of 3.
process such files parallel.
merge files into one mdx file.

Code

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59


import sys
import os


def split_file_by_lines(file_path):
 file_name, file_extension = os.path.splitext(file_path)
 lines_per_file = 28311
 folder_path = "divided_files"

 if not os.path.exists(folder_path):
 os.makedirs(folder_path)
 else:
 raise BaseException("directory exists")

 with open(file_path, "r", encoding="utf-8") as f:
 file_count = 0
 lines = []
 for line in f:
 lines.append(line)
 if len(lines) == lines_per_file:
 new_file_name = f"{file_name}_{file_count:02d}{file_extension}"
 with open(
 os.path.join(folder_path, new_file_name), "w", encoding="utf-8"
 ) as chunk_file:
 chunk_file.writelines(lines)
 lines = []
 file_count += 1
 if lines:
 new_file_name = f"{file_name}_{file_count}{file_extension}"
 with open(
 os.path.join(folder_path, new_file_name), "w", encoding="utf-8"
 ) as chunk_file:
 chunk_file.writelines(lines)
 file_count += 1


def merge_files(files, output_file="res.txt"):
 with open(output_file, "w") as f:
 pass
 for file in files:
 if os.path.isfile(file):
 with open(file, "r", encoding="utf-8") as f:
 content = f.read()
 with open(output_file, "a", encoding="utf-8") as f:
 f.write(content)

split_file_by_lines("./mdx.txt")
os.system(
 r'ls ./divided_files | parallel -j16 "python3 ./prune.py ./divided_files/{} ./divided_files/new_{}"'
)
merge_files([
 os.path.join("./divided_files", f) for f in os.listdir("./divided_files")
 if os.path.isfile(os.path.join("./divided_files", f))
 and f.startswith("new")
])
import shutil


shutil.rmtree("./divided_files")

detailed pruning code:

Show/Hide the code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35


#!/usr/bin/python
import sys
from bs4 import BeautifulSoup
def remove_tags_with_class(html_content):
 soup = BeautifulSoup(html_content, features="html.parser")
 [tag.decompose() for tag in soup.find_all("idm-gs-blk")]
 [tag.decompose() for tag in soup.find_all("pv-gs-blk")]
 def_tags = soup.find_all("def")
 for def_tag in def_tags:
 xhtml_a_texts = [a_tag.unwrap() for a_tag in def_tag.find_all("xhtml:a")]
 [tag.decompose() for tag in def_tag.find_all("chn")]
 new_soup = BeautifulSoup(
 r'<html><body></body></html>',
 "html.parser",
 )
 for tag in def_tags:
 new_soup.body.append(tag)
 for def_tag in def_tags:
 new_tag = soup.new_tag("p")
 new_tag.string = def_tag.get_text().strip()
 def_tag.replace_with(new_tag)
 return str(new_soup)
def process_html_file(input_file, output_file):
 with open(input_file, "r", encoding="utf-8") as file:
 lines = file.readlines()
 with open(output_file, "w", encoding="utf-8") as outfile:
 for line in lines:
 if line.strip().startswith("<head><link"):
 new_line = remove_tags_with_class(line)
 outfile.write(new_line + "\n")
 else:
 outfile.write(line)
input_file = sys.argv[1]
output_file = sys.argv[2]
process_html_file(input_file, output_file)